{ "cells": [ { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "# Lab 9: Neural Networks for text" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "hide_input": false, "slideshow": { "slide_type": "skip" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using Keras 2.2.4-tf\n" ] }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Global imports and settings\n", "%matplotlib inline\n", "import numpy as np\n", "import pandas as pd\n", "import openml as oml\n", "import os\n", "import matplotlib.pyplot as plt\n", "import tensorflow.keras as keras\n", "print(\"Using Keras\",keras.__version__)\n", "os.environ['TF_CPP_MIN_LOG_LEVEL'] = \"2\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Before you start, read the Tutorial for this lab ('Deep Learning with Python')" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Exercise 1: Sentiment Analysis\n", "* Take the IMDB dataset from keras.datasets with 10000 words and the default train-test-split" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "hide_input": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Review 0: ? this film was just brilliant casting location scenery story direction everyone's really suited the part they played and you\n", "Review 5: ? begins better than it ends funny that the russian submarine crew ? all other actors it's like those scenes\n", "Review 10: ? french horror cinema has seen something of a revival over the last couple of years with great films such\n" ] } ], "source": [ "from tensorflow.keras.datasets import imdb\n", "# Download IMDB data with 10000 most frequent words\n", "word_index = imdb.get_word_index()\n", "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)\n", "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", "\n", "for i in [0,5,10]:\n", " print(\"Review {}:\".format(i),' '.join([reverse_word_index.get(i - 3, '?') for i in train_data[i]][0:20]))" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Vectorize the reviews using one-hot-encoding (see tutorial for helper code) " ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "hide_input": true, "slideshow": { "slide_type": "-" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Encoded review: [1, 14, 22, 16, 43, 530, 973, 1622, 1385, 65]\n", "One-hot-encoded review: [0. 1. 1. 0. 1. 1. 1. 1. 1. 1.]\n", "Label: 1.0\n" ] } ], "source": [ "# Custom implementation of one-hot-encoding\n", "def vectorize_sequences(sequences, dimension=10000):\n", " results = np.zeros((len(sequences), dimension))\n", " for i, sequence in enumerate(sequences):\n", " results[i, sequence] = 1. # set specific indices of results[i] to 1s\n", " return results\n", "x_train = vectorize_sequences(train_data)\n", "x_test = vectorize_sequences(test_data)\n", "print(\"Encoded review: \", train_data[0][0:10])\n", "print(\"One-hot-encoded review: \", x_train[0][0:10])\n", "\n", "# Convert 0/1 labels to float\n", "y_train = np.asarray(train_labels).astype('float32')\n", "y_test = np.asarray(test_labels).astype('float32')\n", "\n", "print(\"Label: \", y_train[0])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Build a network of 2 _Dense_ layers with 16 nodes each and the _ReLU_ activation function.\n", "* Use cross-entropy as the loss function, Adagrad as the optimizer, and accuracy as the evaluation matric." ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "slideshow": { "slide_type": "slide" } }, "outputs": [], "source": [ "from tensorflow.keras import models\n", "from tensorflow.keras import layers \n", "\n", "model = models.Sequential()\n", "model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "model.compile(optimizer='RMSprop',\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Plot the learning curves, using the first 10000 samples as the validation set and the rest as the training set.\n", "* Use 20 epochs and a batch size of 512" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "x_val, partial_x_train = x_train[:10000], x_train[10000:]\n", "y_val, partial_y_train = y_train[:10000], y_train[10000:] \n", "history = model.fit(partial_x_train, partial_y_train,\n", " epochs=20, batch_size=512, verbose=0,\n", " validation_data=(x_val, y_val))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1f3/8deZPfueAAkh7JuIKAiiLIKotSqlFhGXIih2c6ldrbXWVv3Vqt381ipqZXGpotYdtSIgLqigIkuAkEAIYc2+z35+f9zJZCEhAZLMJPk8H4953HXuPXMzec+dc++co7TWCCGE6P5MoS6AEEKIjiGBLoQQPYQEuhBC9BAS6EII0UNIoAshRA9hCdWOk5OTdVZWVqh2L4QQ3dKXX35ZrLVOaWlZyAI9KyuLTZs2hWr3QgjRLSml9rW2TKpchBCih5BAF0KIHkICXQgheggJdCGE6CEk0IUQoodoM9CVUk8rpY4qpba1slwppR5RSuUqpbYopc7s+GIKIYRoS3vO0JcBFx9n+beAoYHHTcBjp14sIYQQJ6rN+9C11uuVUlnHWWU2sEIb7fB+ppSKV0r11Vof6qAyCiHaS2vwecDvCQx9jcY94POC3wvaB9rf6KGbTvvbWF5PKUC1Mo4xrWg0rhrK6HWC1wU+lzH0OhvmNRm6m0773IHNmdp4KDCZW16mdQvHoNnrPOYYNFqulLGd4PbNzabbWJZ+Jpx1fYf/+Tvih0XpwP5G04WBeccEulLqJoyzeDIzMztg10KEmWBY1RkB5AkMW5r2BB5eJ3hqweNsNL9+PLDM62y6fmuhrX2hPgKiPVxVYRvo7aa1fgJ4AmD8+PHSs4boWFobZ2/umobw89QGhs3m+dxGAPrczcY9zcabL3cfP7C9zqZnsKFgsoDJCmarcVYYHLc0DJUZTG2d4R7nzBcFaOOYw7HjEJhuYRzAbAeLDSwOsNgDw8bj9sA69hbm24wyNDmD1i2cTbey3O9rdhatjvNaW1qump7h+xvtL1imNpYlZHXKn74jAv0A0L/RdEZgnhDt5/OAswLqyqGuDJzlxnjzYV0ZuCobhXZto9CuDX2YghGYlggjfKwRzQLLAdb6YURgeYQxzxrZ8nxLYDo47jBCzVQf0pamoR2s7hC9TUcE+hvAzUqpF4CJQIXUn/dy7lqoLYbaEqgpaTQeGNaWHBvW7uqO2bfZFgi/yMAwqiEgg/MijPXMNiMET2jcYgyPCexGwW0OWRNJopdr852nlPoPMB1IVkoVAr8HrABa68eBVcAlQC5QCyzsrMKKEHJVQ0UhVOyHygNNw7mmOBDapca4t+7Et69M4IgDRzxEJEBEfGA8/th5jjiwRbcQ1JESpqJXa89dLvPbWK6Bn3RYiUTX09oI4ooCI7TL9xvBXVEI5QXGeF1Z+7dntkNUMkQmQmQyRCYFppMCj0QjoBsHtS3GqNMVQpw0OZ3pqfx+o645WB9d1lAHXVtqhHd5ILQrCts+qzbbIS4D4vtDbAZEpwTCuT6wkxqmbVFSjytECEigdyeuKji6A0pyjVAOBnVZs4uJZcYFxhO5QOiIh7j+RmAHhxkQl2mMR6VISAsR5iTQw5HPC6V5cGQbHMmGo9nGeHnBiW3HFhOo0ohrWsURmQix6RCfaYR3XAY4YjvntQghuowEeihpDdVH4Mh241Ef3EU5xq/nmjPbIGU4JA83zpjr65+b10dHJBgXDs3Wrn9NQoiQkUDvClpD1WEo2Q3Fu6E4pyHE60pbfk58JqSOhrTRkDYK0k6DxMFyF4cQolWSDh3JUwcleY2Ce3dgPBfcVS0/xxHXNLhTR0PqSKkCEUKcMAn0E6U1VB0yzrKLdxsXKOvDu2I/TX7e3FhEAiQNheTAIzUQ4LHpcrFRCNEhJNDbw+eBgg2w6x3YtQrK8ltez2SBhIFGYCcNCYT3MCPIo5K6tMhCiN5HAr01deWQu9oI8dz3jdsA6zniIGUkJA9pCOzkoUaDO3IhUggRIhLojZXlw653jbPwfZ8Y7UbXSx4Ow79lPDImGK20CSFEGOndge73w8GvjADf9Y5x22A9ZYasKTDsYiPEkwaHrpxCCNEOvS/QfV7Y/T8jxHPeg5qjDcvssTBkJgy/BIZcYPwARwghuoneF+j/vRG2v9owHZfZUJUy4Fyj0X0hhOiGeleg73jTCHNbNJz3U+NMPHWU3DYohOgRek+gOytg1S+N8QvugbMXh7I0QgjR4XpPoH/wR3wlh6lxn4ZzfQX6gwfQPh/a5wWvD+3zgc+Lbjzu8R473+tFozFFRGKKDDyiohrGIyMxRUU2nY6MREVGYoqMMpbZ7ca+vV5jez4f2usDryc4v8kyj9coQ/04GlNExLHbj4pCWa0o+cYhRK/UowNda417716qX11B9WuvUlvcB3Qp8O9QF63zWCzHfJg0/bCJQjkcxrqBDmy11uDXwY50m0z7/WjtN34A6/cHO/tVVivKZkNZbQ3jttbGrcFxU2CZKTraeERFYY6ORlnl/n0hTlWPC3S/203txo1Ur/uQ6g8/xFNQ3+SsHUyKyPHjiZw40Qg2sxksZpTZgrKYwdxs3GJtcT6ArqvDX1vb8KipbTodfNTgr61FN17H6USZzSiLBSwWlMVilMVqMfbTeLylZUrhdzqbbN9fU4u/pga8XvyVlfgrK0P3RzgJym4PhHwU5qjohsCPNgLfFNVoOjYWW2YmtoEDMcdKmzdC1OsRge4tKqJ6/Xqq162j5pNP8dfWBpeZo+xEJZcRMyyOqHs+wJyUEsKSdj7tdrfywdLwgaKddcaFYGUyhiZlVNM0njaZAGV0C6cwpuvX0dqoFnK7jYfHc+y4p2Hc73aj3R5jntuDdrnw19Tgr67GFxhqlwufy4WvpATPCbxec0oy9qyB2AYNwj7IGNoGDsLar2/gNZwav9uN92gR3qNH8B49ivfoUXwVldiysnCMHoVtwADjQ7aTab8fz/79OHfswJWXh3Z7Gr5h+fwN36T8Gvw+tD8w3mg52o/2G9+6lMUcOJmwGicNFgvKagFr/bS1YV79cosVZbUY37bsdkwREccdStVf1+uWga79fpzZO6het47qdetwbtvWZLl9+HCip00jetwQItbfgNIuWPAM9PAwB1A2G2abDXN8fKiL0m5aa7TTaQR8dTX+6hr8NdVNp6ur8dcY076yctz5+bjz8/EVFVNbVEztxo1NtqnsdmxZWdgGDcQ+cBC2gQON8awsTFFRaL/f+PA4ehTvkaOBsD5iTDea5ys7fl+qKiICx4gROEaNwjFqJI5Ro7APHoyynfztr9rjwbVnL87sbJw7snFl78C5cyf+6uqT3mYoKIcDk92OiojA5HAY04GhUQ0XqJZr62Frvp7xAUML36aV2dxsvqXpt2FzYP6pftgo1bAdkwlMJmPcbG4Ybzy/i3S7QD/ywJ+pfPttvEVFwXnKbidq0iSip08jeto0rP36GXW9yy8D7YIzroWBU0NYanE8Sinjnz4iAktK+z90td+P99AhXHv24t67B9fevbj37MW9Zw/eoiJcu3bh2rWL5g0XmxMS8FVVgdfb4nabrmzGkpKCJS0Va2oqlpRUTDExuPJycWZn4z14iLqvv6bu668bXo/Vin3oUByjR2EfOZKIUaOwDx+OKSLimM37nU5cu3bh3LEDZ/YOnNnZuHJy0G73MetaUlKwjxqJY9gwTJGRRvMT9d+mlAllNoZGkChU4+VN1lXGRX6vx7jw7jEuwmuPB+31NFyQr59fP8/jMeZ5PPjdLnSdE7/L2WzoQtfVGd/SnE58TidUVBzzWnqVxuEeGMZeeCH9/vxAh++q2wW65/BhvEVFWPr0CQZ41KRJx/6zbH4O8j8yOi2+8N7QFFZ0KmUyYU1Px5qeDlPOa7LMV1WFe+9e3Hv3GoG/Zw+uvXtw7ysInnWbExKwpKVhSU3BmpaGJSUVS2oqljRjaE1Lw5yYeNyqG29ZGa4dRhAbjx248/OD00EmE7ZBA3GMMqpp3Pv24dqxA1feHqM6pBlr//7GWf/IkcaZ/8iRJ/RhF2ra5zOq1pxO49tXs6H2eFp+uFuZ32Qdt1GtFLgLrfGdasaHlO/489vzQd4Wf301l24YDwyD832+wI0GgXGfL9i4tvacSMVi+ymtW2m/u5ONHz9eb9q06YSf59y1C7TGPnx4619lqovgn+ONDpO/+xScPvcUSyt6Cu314i0txRwfj+kUqkWOx1ddjWvnzuAZtzM7G1denvFP3ZzZjH3QoIbqmpFGeJtjYjqlbKJrad3obrFGgY/JhKn+brMTpJT6Ums9vqVl3e4M3TF8eNsrvfcbI8wHz4Qx3+v8QoluQ1ksWFNTO3Uf5uho426q8Q3/c36XC1dOjnEGv28ftsxMHKNGYh827KT/sUX4M242MKpcuqImvdsFept2r4atL4ElAi79q/ysX4QFk91OxJgxRIwZE+qiiB7s1O/rCifuGnj7dmP8/DuNDieEEKKX6FmBvu5PUF4AfcbApB+HujRCCNGlek6gH9wMGx41btu67BEw97zaJCGEOJ6eEeg+L7x5G2g/TPwRpJ8Z6hIJIUSX6xmB/sUSOLQZ4vobdedCCNELdf9AL9sHa+4zxr/9V7BHh7Y8QggRIu0KdKXUxUqpXUqpXKXUHS0sz1RKrVVKfa2U2qKUuqTji9oCreHtn4OnFkZ/F4Zd2CW7FUKIcNRmoCulzMCjwLeAUcB8pdSoZqvdBazUWo8DrgL+1dEFbdG2VyD3fXDEwcUd3y6CEEJ0J+05Qz8byNVa79Fau4EXgNnN1tFAfcPUccDBjitiK2pL4d3Al4VZ90JMWqfvUgghwll7Aj0d2N9oujAwr7F7gGuVUoXAKuCWljaklLpJKbVJKbWpqFFriSfl/buhpggGnAvjrju1bQkhRA/QURdF5wPLtNYZwCXAM0qpY7attX5Caz1eaz0+5VRajtv7EXz9DJhtcOnfjaYphRCil2tPEh4A+jeazgjMa+wGYCWA1noD4ACSO6KAx/A44a2fGuNTfgEpwzplN0II0d20J9A3AkOVUgOVUjaMi55vNFunAJgJoJQaiRHop1in0oqP/gIluZA8HM77aafsQgghuqM2A11r7QVuBt4DdmDczbJdKfVHpdTlgdV+DixWSn0D/Ae4XndGQ+t+H+R/bIxf9g+w2Dt8F0II0V11uw4ugqE+aFrHF0oIIcLc8Tq46H5XE01mCXMhhGhB9wt0IYQQLZJAF0KIHkICXQgheggJdCGE6CEk0IUQooeQQBdCiB5CAl0IIXoICXQhhOghJNCFEKKHkEAXQogeQgJdCCF6CAl0IYToISTQhRCih5BAF0KIHkICXQgheggJdCGE6CEk0IUQoosU1Rbxr83/4tMDn3bK9i2dslUhhBAAaK3ZUryF53c8z//2/Q+v38vEvhOZnD65w/clgS6EEJ3A7XPzXv57PLfjObaXbAfApEzMGjCL+SPmd8o+JdCFEKIDHa09yspdK3kp5yVKnaUAxNnj+N7Q7zFv+Dz6RvfttH2HVaB7PB4KCwtxOp2hLooAHA4HGRkZWK3WUBdFiLCmteabom94fsfzvL/vfbzaC8DwhOFcPfJqLhl4CQ6Lo9PLEVaBXlhYSExMDFlZWSilQl2cXk1rTUlJCYWFhQwcODDUxREiLLl8Lt7d+y7P73ye7JJsAMzKzKwBs7hm5DWcmXpml2ZZWAW60+mUMA8TSimSkpIoKioKdVGECDtHao7w4q4XeWX3K8FqlXh7PN8bZlSr9InqE5JyhVWgAxLmYUT+FkI08Gs/nx36jP/u/i+r963Gp30AjEgcwdUjruZbA7/VJdUqxxN2gS6EEOFkf9V+Xs99ndfzXudwzWHAqFa5KOsirh5xNeNSx4XNyY8EejPR0dFUV1eHuhhCiBCq9dSyumA1r+W+xsbDG4Pz06PTmT14NnOGzglZtcrxSKALIQQNd6q8lvsa7+a/S42nBgCH2cEFAy5gzpA5jO8zHpMK3x/Yh22gZ93xdqdsN/+Bb7drPa01v/rVr3jnnXdQSnHXXXcxb948Dh06xLx586isrMTr9fLYY48xefJkbrjhBjZt2oRSikWLFnH77bd3SvmFEB3raO1R3sx7k9dyXyO/Mj84//SU05kzZA4XZV1EjC0mdAU8AWEb6KH23//+l82bN/PNN99QXFzMhAkTmDp1Ks8//zwXXXQRv/3tb/H5fNTW1rJ582YOHDjAtm3bACgvLw9x6YUQx+PxeVhXuI7Xcl/j4wMf49d+AJIjkrls0GV8Z8h3GBQ/KMSlPHFhG+jtPZPuLB9//DHz58/HbDaTlpbGtGnT2LhxIxMmTGDRokV4PB6+853vcMYZZzBo0CD27NnDLbfcwre//W0uvPDCkJZdCNGygsoC/rPzP7y15y3KXcaJl0VZOD/zfOYMmcO56ediMYVtLLapXZVBSqmLlVK7lFK5Sqk7WlnnSqVUtlJqu1Lq+Y4tZviYOnUq69evJz09neuvv54VK1aQkJDAN998w/Tp03n88ce58cYbQ11MIUQjW4u28rN1P+Oy1y7j2R3PUu4qZ2jCUH414Vd8cOUH/P38vzOt/7RuHebQjjN0pZQZeBSYBRQCG5VSb2itsxutMxT4DXCu1rpMKZXaWQXuKlOmTGHJkiUsWLCA0tJS1q9fz0MPPcS+ffvIyMhg8eLFuFwuvvrqKy655BJsNhtXXHEFw4cP59prrw118YXo9fzaz8cHPmbptqVsOrIJAIvJwmWDLmP+iPmMShoVNrcbdpT2fBydDeRqrfcAKKVeAGYD2Y3WWQw8qrUuA9BaH+3ogna1OXPmsGHDBsaOHYtSigcffJA+ffqwfPlyHnroIaxWK9HR0axYsYIDBw6wcOFC/H6jHu5Pf/pTiEsvRO/l8Xl4e+/bLN++nNzyXACirdHMHT6Xa0ZcQ1pUWohL2HmU1vr4Kyj1PeBirfWNgenrgIla65sbrfMakAOcC5iBe7TW77awrZuAmwAyMzPP2rdvX5PlO3bsYOTIkaf0gkTHkr+J6C6q3FW8nPMyz+54lqO1xjllakQq1426jiuGXdFt7lRpi1LqS631+JaWdVSFkQUYCkwHMoD1SqkxWusmt3torZ8AngAYP3788T9JhBCiHY7UHOG5Hc/xUs5LVHuMHwUOiR/C9aOv55KBl2A1957WQtsT6AeA/o2mMwLzGisEPtdae4C9SqkcjIDfiBBCdILcslyWbV/G23vfxus3mqsdnzaehactZEr6lB5XP94e7Qn0jcBQpdRAjCC/Cri62TqvAfOBpUqpZGAYsKcjCyqEEFprvjzyJUu3L2V94XrA6AXowgEXsvC0hZyWfFqISxhabQa61tqrlLoZeA+jfvxprfV2pdQfgU1a6zcCyy5USmUDPuCXWuuSziy4EKL38Pl9rN2/lqXblrKleAsAdrOd7wz5DgtGLaB/bP82ttA7tKsOXWu9CljVbN7djcY18LPAQwghOoTL5+LNvDdZvn158Gf58fZ4rhpxFfNHzCfRkRjaAoaZ7n0XvRCiR6p0V7Jy10qe2/EcxXXFAPSL6sf3R3+fOUPmEGmNDHEJw5MEuhAibBypOcIz2c/wUs5L1HprAaNfzkWnLeLCrAu7/S85O5scnRDxer1YLHL4hQDIK89j6balTe5YmdhnIotOW8Q5/c7plXesnIzwTZR74jppuxVtrvKd73yH/fv343Q6ue2227jpppt49913ufPOO/H5fCQnJ/PBBx9QXV3NLbfcEmw29/e//z1XXHFFk04yXn75Zd566y2WLVvG9ddfj8Ph4Ouvv+bcc8/lqquu4rbbbsPpdBIREcHSpUsZPnw4Pp+PX//617z77ruYTCYWL17M6NGjeeSRR3jttdcAeP/99/nXv/7Fq6++2jnHSYgu8NWRr1i6bSnrCtcBDXesLDptEaOTR4e2cN1Q+AZ6CD399NMkJiZSV1fHhAkTmD17NosXL2b9+vUMHDiQ0lKjU9h7772XuLg4tm7dCkBZWVmb2y4sLOTTTz/FbDZTWVnJRx99hMViYfXq1dx555288sorPPHEE+Tn57N582YsFgulpaUkJCTw4x//mKKiIlJSUli6dCmLFi3q1OMgRGfwaz/r9q9j6balbC7aDDTcsfL9Ud8nMzYzxCXsvsI30NtxJt1ZHnnkkeCZ7/79+3niiSeYOnUqAwcOBCAx0biyvnr1al544YXg8xISEtrc9ty5czGbzQBUVFSwYMECdu/ejVIKj8cT3O4Pf/jDYJVM/f6uu+46nn32WRYuXMiGDRtYsWJFB71iITpfpbuSN/Pe5MVdL7K3Yi8AsbZYrhpxFVePuJqkiKQQl7D7C99AD5F169axevVqNmzYQGRkJNOnT+eMM85g586d7d5G4/o+p9PZZFlUVFRw/He/+x3nn38+r776Kvn5+UyfPv242124cCGXXXYZDoeDuXPnSh28CHtaa7YUb+GlXS/xXv57OH3G/0OfqD58f9T3uWLoFXLHSgcK387xQqSiooKEhAQiIyPZuXMnn332GU6nk/Xr17N3r3FWUV/lMmvWLB599NHgc+urXNLS0tixYwd+v/+4ddwVFRWkp6cDsGzZsuD8WbNmsWTJErxeb5P99evXj379+nHfffexcOHCjnvRQnSwKncVL+x8ge+9+T2uXXUtr+e9jtPnZGKfiTw07SFWfXcV1426TsK8g0mgN3PxxRfj9XoZOXIkd9xxB5MmTSIlJYUnnniC7373u4wdO5Z58+YBcNddd1FWVsZpp53G2LFjWbt2LQAPPPAAl156KZMnT6Zv376t7utXv/oVv/nNbxg3blwwvAFuvPFGMjMzOf300xk7dizPP9/QX8g111xD//79pQVEEXa01mwr3sbvP/09M1+ayf2f309OWQ4J9gSuH309b815i6cueoqLsy7Gauo9DWZ1pTabz+0s48eP15s2bWoyrz1NtWqtqXF5ibJbeuWtTDfffDPjxo3jhhtu6JL9SfO5oi01nhre3vM2L+e8zI7SHcH549PGM3fYXC4YcAE2sy2EJexZuqL53C6TW1RNndvH4JRoouzdrvin5KyzziIqKoq//OUvoS6KEGSXZPNSzkus2rMq+COgOHscswfP5ophVzAorvt1stzddbtEjLZbqHP7KKtx97pA//LLL0NdBNHL1XhqeHfvu7yU8xLbS7YH55+ZeiZzh89l1oBZ2M32EJawd+t2iZgQaaOoykV5nYe+fo3Z1PuqXYToSj6/j88Pf84beW/wwb4PgneqxNhimD14Nt8b9j0Gxw8OcSkFdMNAd1jNRNks1Li9VNS5SYySswEhOsOe8j28nvc6b+15K9ilGxhn41cMu4ILB1yIw+IIYQlFc90u0AESomzUuL2U1ngk0EW3orUmpyyHQfGDwvJOj3JnOav2ruLNvDfZVrItOD8jOoPLB1/OpYMvpX+MtD0errploMdFWDlUrqh1e3F6fDis5lAXSYg2+fw+fvvJb3l7z9v0i+rHgtELmDN0DhGWiJCWy+PzsP7Aet7Me5MPCz8MNo4VbY3moqyLuHzw5YxLHdcr7yrrbrploJtNirhIK6U1bkpr3PSLD80/RONGuJrLz8/n0ksvZdu2bS0uF72LX/u5Z8M9vL3nbQAO1hzkT1/8ice/eZxrRl7DVSOuIs7eSQ3StUBrTXZJNm/kvcE7e9+hzGX8KM6kTJybfi6zB8/m/P7nS5VKN9MtAx0gMcpGaY2b8loPfeIcmOTsQYQprTX3f3Y/r+W+RoQlgkdnPkqlq5Kntj7FtpJt/HPzP3l629PMHTaX60ZdR1pUWqeVJb8inzX71/BG7hvkVeQF5w+JH8LswbO5ZNAlpEamdtr+RecK20Afs3xMp2x364KtrS6744476N+/Pz/5yU8AuOeee7BYLKxdu5aysjI8Hg/33Xcfs2fPPqF9Op1OfvSjH7Fp0yYsFgt//etfOf/889m+fTsLFy7E7Xbj9/t55ZVX6NevH1deeSWFhYX4fD5+97vfBX+ZKrofrTUPbnyQlTkrsZlsPDLjESb0mQDAjMwZbDy8kX9v+zefHvyU5dnLeW7nc1w++HKuH309A+MGnvL+XT4XXx7+kvUH1vNR4UcUVBUElyU6Erlk4CVcNvgyRiaOlCqVHiBsAz0U5s2bx09/+tNgoK9cuZL33nuPW2+9ldjYWIqLi5k0aRKXX375Cb35H330UZRSbN26lZ07d3LhhReSk5PD448/zm233cY111yD2+3G5/OxatUq+vXrx9tvG1/NKypC1+qkODVaa/721d94dsezWE1W/n7+35nUd1JwuVKKs/uezdl9zya7JJuntz3N+/ve57+7/8uru19lZuZMbhhzwwn3ZH+45jDrC40A//zw59R564LL4uxxnNvvXC7OupjzMs4Lywuz4uSFbaAf70y6ntfnZ8fhKrTWjOgTg81yahdHx40bx9GjRzl48CBFRUUkJCTQp08fbr/9dtavX4/JZOLAgQMcOXKEPn36tHu7H3/8MbfccgsAI0aMYMCAAeTk5HDOOedw//33U1hYyHe/+12GDh3KmDFj+PnPf86vf/1rLr30UqZMmXJKr0mEzr+++RdLty3Foiw8PO1hpmS0/rcclTSKh6c9zL7KfSzbvozXc19ndcFqVhesNnruGbOIc/q23HOP1+/lm6JvjBA/8BG7y3Y3WT4icQRT0qcwNWMqY5LHYDbJTQQ9VdgGentYzCbiHFbK69yU1XpIiz31N+rcuXN5+eWXOXz4MPPmzeO5556jqKiIL7/8EqvVSlZW1jFN4p6sq6++mokTJ/L2229zySWXsGTJEmbMmMFXX33FqlWruOuuu5g5cyZ33313h+xPdJ0ntzzJ4988jkmZeGDqA8zInNGu5w2IHcDvz/k9Px77Y57Z8Qwrd63k88Of8/nhzxmZOJIbxtzABZkXUOGu4OMDH/NR4Ud8cvATqtxVwW1EWiI5p985TEmfwnnp53VqnbwIL9060AESowKBXuMmNcZ+yvWA8+bNY/HixRQXF/Phhx+ycuVKUlNTsVqtrF27ln379p3wNqdMmcJzzz3HjBkzyMnJoaCggOHDh7Nnzx4GDRrErbfeSi13f3oAACAASURBVEFBAVu2bGHEiBEkJiZy7bXXEh8fz1NPPXVKr0d0veXbl/PI14+gUPy/8/4fF2VddMLbSIlM4Wdn/Ywbx9zIyl0reSb7GXaU7uAXH/6CREciZc4yNA0N62XFZjE1YypTMqZwVupZWM1SldIbdftAj7JbsFlMuL1+ql1eYhyn9kYePXo0VVVVpKen07dvX6655houu+wyxowZw/jx4xkxYsQJb/PHP/4xP/rRjxgzZgwWi4Vly5Zht9tZuXIlzzzzDFarlT59+nDnnXeyceNGfvnLX2IymbBarTz22GOn9HpE13p+x/M8vOlhAP4w+Q98e9C3T2l7sbZYbhxzI9eOvJY38t5g6balFFYXYjPZmNB3glGVkj6V/rHyYx/RDZvPbcmRSidHKp3ERVgZkBTV9hNEu0nzue33cs7L/GHDHwD43aTfceXwKzt8H16/l7zyPPrH9JfOIXqp4zWf2yM6uEiItKGASqcXr88f6uKIXuj13Nf544Y/AvDrCb/ulDAHsJgsDE8cLmEuWtTtq1wAbBYT0Q4rVU4PZbUeUmK6rn2XrVu3ct111zWZZ7fb+fzzz7usDCK03tn7Dnd/ejcaze1n3c61o64NdZFEL9UjAh2MX44age4mOdrWZT+SGDNmDJs3b+6SfYlT98WhL3h6+9MMjR/KhD4TODP1TKJt0Se9vdX7VvObj36DX/v5yRk/YdFpizqwtEKcmB4T6DEOCxaTCafHR63b1+s6vxBtW1uwlp9/+HM8fg+fHPiEZduXYVImRiWOYkKfCUbAp51JlLV912E+3P8hv1z/S3zax+Ixi/nB6T/o5FcgxPH1mNQzKUVClJWiKhdltb2vNyNxfKv2rOLOj+/Ep33MGTKH5Ihkvjj8BduLt7OtZBvbSraxdPtSzMrMqKRRjO8znglprQf8pwc+5fZ1t+P1e1kwagG3jLtFfjovQq5dqaeUuhj4B2AGntJaP9DKelcALwMTtNabWlqnMwV7M6r10DdOejMShldyXuEPG/6ARnPjmBu5ddytwfCt9dSy+ehmNh7ZyMbDG9levJ2txVvZWryVpduMgB+dNNoI+EAVzbbibdy69lY8fg/zR8zn5+N/LmEuwkKbga6UMgOPArOAQmCjUuoNrXV2s/VigNuAkF0NlN6MRHPPZD/DgxsfBOC2M2/jxjE3NlkeaY1kcvpkJqdPBoyA//ro12w8vJGNR4yA31K8hS3FW3h629OYlRmTMuHxe7hi6BXccfYdEuYibLTnDP1sIFdrvQdAKfUCMBvIbrbevcCfgV92aAlPUFf2ZnS89tBFaGmtWbJlCY9ufhSAO86+g2tGXtPm8yKtkZybfi7npp8LGJ0i1wf8psOb2F6yHY/fw+WDL+fuc+7GpHrEnb+ih2hPoKcD+xtNFwITG6+glDoT6K+1flsp1emB7tf+Vv+RemNvRl6vF4tFrhnU01rzty//xtLtSzEpE/eccw9zhs45qW1FWaM4L/08zks/DzACvqCygBGJI+TMXISdU04BpZQJ+CtwfTvWvQm4CSAzM/O46+4YcfK/TjQFHntbWDZy545Wn9eR7aFXV1cze/bsFp+3YsUKHn74YZRSnH766TzzzDMcOXKEH/7wh+zZsweAxx57jH79+jXp9ejhhx+murqae+65h+nTp3PGGWfw8ccfM3/+fIYNG8Z9992H2+0mKSmJ5557jrS0NKqrq7nlllvYtGkTSil+//vfU1FRwZYtW/j73/8OwJNPPkl2djZ/+9vf2n+Qw5Rf+7n/s/tZmbMSi7LwwNQHTqotldZEWaMYmSS/nBXhqT2BfgBo3FBERmBevRjgNGBd4IylD/CGUury5hdGtdZPAE+A8dP/Uyh3p+jI9tAdDgevvvrqMc/Lzs7mvvvu49NPPyU5OZnS0lIAbr31VqZNm8arr76Kz+ejurqasrKy4+7D7XZT33xCWVkZn332GUopnnrqKR588EH+8pe/cO+99xIXF8fWrVuD61mtVu6//34eeughrFYrS5cuZcmSJad6+ELO6/dy9yd38+aeN7GZbPzt/L8xNWNqqIslRJdpT6BvBIYqpQZiBPlVwNX1C7XWFUBy/bRSah3wi1O9y6W1M2mf30d+ZT5OrxO72U5WXBYWU9OXobVm99FqnB4fAxIjiYu0tWufHdkeutaaO++885jnrVmzhrlz55KcbByyxMREANasWcOKFSsAMJvNxMXFtRnojXsyKiwsZN68eRw6dAi3283AgUZvN6tXr+aFF14IrpeQkADAjBkzeOuttxg5ciQej4cxYzqnh6iu4va5+fX6X7O6YDURlgj+OeOfnN337FAXS4gu1eYVHa21F7gZeA/YAazUWm9XSv1RKXV5ZxewObPJzIDYAdjNdlw+F/sq9+Hz+5qso5QiMcoI8dJazwltv7499BdffPGY9tA3b95MWlpau9pDP9nnNWaxWPD7G9qmaf78qKiG+6NvueUWbr75ZrZu3cqSJUva3NeNN97IsmXLWLp0KQsXLjyhcoWbOm8dt665ldUFq4mxxfDkhU9KmIteqV2X6LXWq7TWw7TWg7XW9wfm3a21fqOFdad39j3oFpOFAbEDsJltOL1O9lUdG+rxEVaUUlQ5Pbi97W+wa968ebzwwgu8/PLLzJ07l4qKipNqD721582YMYOXXnqJkpISgGCVy8yZM4NN5fp8PioqKkhLS+Po0aOUlJTgcrl46623jru/9PR0AJYvXx6cP2vWLB599NHgdP1Z/8SJE9m/fz/PP/888+fPb+/hCTvV7mp++P4P+eTgJyQ6Ell60VLGpowNdbGECIlue8+V1WxlQOwArCYrdZ46CqoK8OuG4K7vzQigrNbd7u221B76pk2bGDNmDCtWrGh3e+itPW/06NH89re/Zdq0aYwdO5af/exnAPzjH/9g7dq1jBkzhrPOOovs7GysVit33303Z599NrNmzTruvu+55x7mzp3LWWedFazOAbjrrrsoKyvjtNNOY+zYsaxduza47Morr+Tcc88NVsN0N+XOchb/bzFfHf2K1MhUll68lOGJw0NdLCFCptu3h+72udlbsRev30u0LZr+Mf2DtzRWOT3sLa7BZjYxvE+M3GbWzKWXXsrtt9/OzJkzW10nXNtDL64rZvH/FpNbnktGdAZPXvgkGTEZoS6WEJ2uR7eHbjPbyIrNwmwyU+2uprCqMHimHm23YDObcPuM3oyEoby8nGHDhhEREXHcMA9Xh6oPseCdBeSW5zI4bjDLv7VcwlwIekjjXHaLnazYLPZW7KXKXcXB6oOkR6ejlCIhysaRSielNe5T7p6uJd2xPfT4+HhycnJCXYwT4vF52F2+m+0l23lyy5McqjnEyMSRLJm1hARH96wyEqKjhV2ga61PqmrEYXEwIHYA+yr3UeGqQClFv6h+JETaOFrpDPZmZDF37JeSntweeqiq4zw+D7nluWSXZLO9ZDvZJdnklOXg8TfcsXRGyhn864J/EWOLCUkZhQhHYRXoDoeDkpISkpKSTirUI62RZMZmsq9yH+XOckyY6BPVJ2S9GXVnWmtKSkpwOByduh+P30NeeZ4R3sVGeO8q29UkvOtlxWYxKmkUp6eczpwhc6QbNiGaCatAz8jIoLCwkKKiolPajtvnprSulMMcpsBagFVFUVLjptSsSIvt3IDqSRwOBxkZHVs3fbjmMBsObgieee8q3YXbf+xdSFmxWYxMGsnopNGMShrFyMSRp9SzkBC9QVgFutVqDf7C8VSt27+O29fejld7+cGYH7H0g6GU1Lh59ceTGZcpda5dRWvN7vLdrC1Yy5r9a8guad5IJ2TGZAaDe3TyaEYkjpCqFCFOQlgFekea3n86f5r6J369/tcs2foYZ4y8lo82ncaLG/dLoHcyn9/H5qLNrClYw5qCNRRWFwaXRVgimNR3EuNSxxln3kkjibXFhrC0QvQcPTbQAS7OuhiX18Vdn9zF5ppnsSbM5s1vzPzu0lHSRV0Hc3qdbDi4gTX71/Dh/g8pczW0Q5PoSGRaxjRmZM5gUt9JOCxS7SVEZ+jxqTZ7yGxcPhf3fnYvjj6vU3fQyttbRnPlhP5tP1kcV7mznA8LP2RNwRo2HNpAnbcuuKx/TH9m9J/BjMwZjE0Zi9nU89ulFyLUenygA1w5/ErqvHU8vOlhHH1f4cmv47lywm2hLla3orWmxFnCvsp9ZJdks3b/Wr488mWT5hZGJ41mRuYMZvSfweD4wfLLXCG6WK8IdIAFoxdQ6arlia3/4pDt38x9bRNzhn+LmZkzSYtKC3XxwkaFq4J9lfuCj4LKAvIr8ymoKqDGU9NkXYuyMLHvRGZkzmB6/+n0iTp+k8JCiM4VVm25dDatNVe/dC9ba/+LUg2tM56ecjqzMmcxc8BM+sf0/KqYWk8tBVWBoK4saBLg5a7yVp8XY4shKzaLgXEDObffuZyXcZ5c0BSiix2vLZdeFegAHp+fHzz7MesPrCcyIRtrdA5uvyu4fETiCC7IvIBZA2YxKH5Ql5evI3n8HvIr8tldtpvd5buNYdluDtYcbPU5EZYIBsQOIDMmkwGxA5o84u3xUo0iRIhJoDfj9vr54bNfsmbnURKjNT+9zM+2io9ZX7i+SbXCwLiBXJB5ARcMuICRiSPDNsy01hyuOczu8t3klOUEA7y+FcrmrCYrmTGZZMZmkhWbRWZsQ3inRKSE7esUQkigt8jp8bF4xSY+2l1MSoydF2+aRHqilc8OfsbqgtWs3b+WCldFcP306PRguJ+ecjomZUJrjcvnwul14vQ5qfXWGuOBR523jjpfXdNpbx0evweLyYLVZD1m2GTcbMWiLFjNTee7fW7yyvOCAZ5blkuVp+qY16hQZMRkMDR+KEMTGh6ZMZnHdNsnhOgeJNBb4fT4WLRsI5/mldAn1sGLP5jEgCSjWzeP38Omw5v4oOADPij4gOK64uDzIi2RaDROrxNNePR1nWBPYFjCsIbgjh/K4PjB0t6JED2MBPpx1Lq9XP/0Rr7ILyU9PoIXfzCJjISmIejz+/im6BtWF6xm9b7VHKo5FFxmN9txWBw4zA4iLBFEWCKC0w7LsfMiLBFYzVZ8fh8evwev34vH72k67msYb2m5UoqBsQOD4T0sYRhJjpNr0EwI0b1IoLeh2uXl+//+nK8KyumfGMHKH5xD37iIFtfVWlPuKsdutmM32+UHM0KILtWjeyzqCNF2C8sWnc3pGXHsL63j6ic/52ils8V1lVIkOBKItEZKmAshwooEekCsw8oziyYyqm8se4trmP/kZxRVudp+ohBChAkJ9EbiIq08e+NERvSJIa+ohmuf+pzSmmPb6hZCiHAkgd5MYpSNZ2+cyJDUaHYdqeLapz6nvFZCXQgR/iTQW5Acbef5GycyMDmK7EOVfP/pL6h0HtslmhBChBMJ9Fakxjp4fvFEMhMj2VJYwYKnv6DadeyvLoUQIlxIoB9H37gInl88kfT4CL4uKGfh0i+odUuoCyHCkwR6GzISIvnP4kn0jXOwMb+MG5Ztos7ta/uJQgjRxSTQ2yEzKZLnF08iNcbOhj0l3PTMJirqpE5dCBFeJNDbaWByFM8vnkhytI2Pdhcz8y/rWLlpP35/eLTlIoQQEugnYEhqDC/+4BwmZCVQXO3mVy9v4buPfcqWwtY7hRBCiK7SrkBXSl2slNqllMpVSt3RwvKfKaWylVJblFIfKKUGdHxRw8PglGhW/uAc/nHVGaTG2Nm8v5zZj37CHa9soaRaflkqhAidNgNdKWUGHgW+BYwC5iulRjVb7WtgvNb6dOBl4MGOLmg4UUox+4x01vxiOj+YOgizUrywcT/nP7yOFRvy8fr8bW5DCCE6WnvO0M8GcrXWe7TWbuAFYHbjFbTWa7XWtYHJz4CMji1meIq2W/jNJSN596dTmTI0mUqnl7tf386l//cxX+wtDXXxhBC9THsCPR3Y32i6MDCvNTcA77S0QCl1k1Jqk1JqU1FRUftLGeaGpEazYtHZLLnuLNLjI9h5uIorl2zgthe+5kgrrTYKIURH69CLokqpa4HxwEMtLddaP6G1Hq+1Hp+SktKRuw45pRQXje7DBz+fxm0zh2K3mHh980FmPLyOxz/Mw+2VahghROdqT6AfAPo3ms4IzGtCKXUB8Fvgcq11r7066LCauX3WMFb/bBoXjkqjxu3jgXd2cvHf1/NhTs/5ViKECD/tCfSNwFCl1ECllA24Cnij8QpKqXHAEowwP9rxxex++idG8sT3x7N80dkMSo5iT3ENC57+gptWbGJ/aW3bGxBCiBPUri7olFKXAH8HzMDTWuv7lVJ/BDZprd9QSq0GxgD1nW0WaK0vP942w6kLus7m9vpZ+sleHvlgNzVuHzaLidlj+7FgchanpceFunhCiG5E+hQNE0cqnfxp1Q5e23wwOG9CVgILJmdx0eg+WM3yOy8hxPFJoIeZPUXVrNiwj5e/LAw2ydsn1sG1kzK56uxMkqPtIS6hECJcSaCHqWqXl/9+VcjyT/PJK6oBwGY2cenYvlw/OYvTM+JDXEIhRLiRQA9zWms+zi1m+af5fLDzKPV/kjMz41kwOYtvndYXm0WqY4QQEujdSkFJLSs25PPipv1UOY3qmJQYO9dMzOTqiZmkxjhCW0AhREhJoHdDtW4vr359gOWf5pNzpBoAq1nx7TF9+f7kLMb1j0cpFeJSCiG6mgR6N6a1ZkNeCcs+zWf1jiPUN7+eGmNn2rAUpg1PYcqQFOIiraEtqBCiS0ig9xD7S2t59vN9vPb1AY5UNvwY16RgXGYC0wMBf1q/OEwmOXsXoieSQO9htNbsPFzFhzlFrNt1lE35ZXgb9ZyUFGVj6rAUpg1LYeqwFBKjbCEsrRCiI0mg93DVLi+f5BbzYU4RH+4q4kB5XXCZUnB6ehzThqcybVgKZ/SPxyxn70J0WxLovYjWmryiatbtKuLDnCI+31OKu1GHG3ERVqYMTea8IclMHpxMZlJkCEsrhDhREui9WK3by2d7Sli3q4h1u4ooaNYwWEZCBOcOTmbykCTOGZwkt0UKEeYk0EVQfnEN63cX8UluMRvySqgM3Oteb1haNJMHJzN5cBITByURFyF3zwgRTiTQRYt8fk32wUo+zSvmk7wSNu4tpc7jCy43KRiTHsfkIUbAjx+QSITNHMISCyEk0EW7uL1+Nu8v55PcYj7NK+brgvImd8/YzCbOHBDP5MHJnD0wkTP6x+OwSsAL0ZUk0MVJqXF52Zhfyoa8Ej7JK2b7wUoav12sZsXpGfFMyEpkQlYC4wckyg+chOhkEuiiQ5TVuPl8bwkb8kr4Ir+MnYebBrxSMDwtxgj4gYmcnZVInzi5yCpER5JAF52ios7DV/vK+CK/lI17S9lSWNHkFkmA/okRTBhgBPyErEQGp0RJGzRCnAIJdNElnB4fWwor2Jhfyhd7S/lyX1mwA496SVE2zhqQwPA+MQxOiWZwSjSDUqKIsltCVGohuhcJdBESPr9mx6FKNuaXBkK+jOJqV4vr9o1zBAI+iiGpRtAPTo0mNcYuZ/RCNCKBLsKC1pp9JbVs3l9OXlE1uUerySuqJr+49piqmnrRdguDU6KCAV8/PiApSjr9EL3S8QJdvueKLqOUIis5iqzkqCbzvT4/hWV15BUZAZ93tMYI/KJqyms9fFNYwTeFFU2eYzEZ2xqaGs3Q1GiGpMUwNDWagclRciul6LXkDF2EtZJqF3lFNYGgrw4GfWFZHS29dU0KBiQZ1TZDU6MZmhbN0FSjvl5+FCV6AqlyET1OndsXrLbJOVLF7qPG+L6SGvwtvKWVMtqtGZoaw5DUaPrFOegT5yAt1kHfuAiSo21YzFKFI8KfVLmIHifCZua09DhOS49rMt/p8bG3uMYI+EDQ7z5aTX5xDftL69hfWseanUeP2Z5JGX239ok1gr5PrIO0wLBPo2GkTf5lRPiSd6foURxWMyP7xjKyb2yT+W6vn30lRtDvKarmUIWTI5VODlc6OVzhorjaxZFK49G8vr6xGIeFvoEz+9QYB2mxdtJijWFqbP18O1Y52xchIIEuegWbxcTQtBiGpsW0uNzt9XO0KhDyFS4OVdQFAt/FkYpA8Fc6qXJ6qXJWBzvubk1ytK1J4KcGQj8txkFqrJ3UGAdJ0TYJftGhJNCFwAj8jIRIMhJa7/BDa01ZrYfDFU6OVDk5WukMnNUbw/oPhKIqF8XVboqr3WQfOv5+EyKtpMTYSYmxkxxtJyXa3nQ6MJ4QaZOepkSbJNCFaCelFIlRNhKjbIwittX1fH5NSaAK53ClEfLB8K8yhkVVLkprXJTVeiir9bR5xm82GftOibaTHGMnKVCO+kdCpI2k6MAwykZchFU6Cu+FJNCF6GBmkyI1UM0yhrhW1/P5NaU17sAZvRHyRYFhcDowXlbrCU7Txlk/GBd54yMDgR8YJkQZYZ8QZSPWYSE2wkqsw0qMw0JcYDzaYZFvAt2YBLoQIWI2qWCVSlvcXj8lNS6Kq9wUVTsprfFQWuM6ZlhW66Gk2kWl00tpjZvSGvcJlyvabgkGfozDQqzD2mQ8xmEhwmbGbjFhtwSG1kbjFnNg+tjl8mHRuSTQhegGbBYTfeMi6BsXAcc566/n8fkpq3VTVuMJBntprZvSajdltW4q6zxUOj1UOr1U1nmocnqpdBrDapfxOFjh7PDXYTEpHFYz8ZFWkqLtJEcZVUVJ0UY1UlK0jaQoO0nRNpKjjWsH0sRD+7Ur0JVSFwP/AMzAU1rrB5ottwMrgLOAEmCe1jq/Y4sqhGgvq9lEaozjhDv99vk11S4vVU4PlXVGyDcO/Mo6Y5nL68fl9RlDT6Px+vke/zHrOL0+vIHtV7u8FJbVtatMsQ4LydH2YNgnRNkwm8DnNy5U+7XGr8GvNTow9Gvw++uXGdO60Xp2i4kIq5kImxmH1WyMB6YjbA3TDluzZVbj2wcafFrj82v8fvD6/fi1xuc3jqHPrxuWB4aN56dE24/5DUVHaDPQlVJm4FFgFlAIbFRKvaG1zm602g1AmdZ6iFLqKuDPwLwOL60QolOZTYq4CKvROXhCx25ba43Xr6l1+yivNe4CKql2UVrjpqTGTXG1i5JqNyU19UPjm0Wl00ul08ue4pqOLVAIXTa2H/83f1yHb7c9Z+hnA7la6z0ASqkXgNlA40CfDdwTGH8Z+KdSSulQtSsghAg7SimsZkVchIm4CCsDkqLafI7fr6mo8xjXD6rdwfBHa5RSmE0KkzK2bVLGuEkpVGDY0nIwrknUeXzGw+3DGRw35jsD8+vXaTzt9PgAhdkEZqUwmxVmpTCZjKHZ1PAwNZ6uL49JMbJvy7+HOFXtCfR0YH+j6UJgYmvraK29SqkKIAkobrySUuom4CaAzMzMkyyyEKK3MJkUCYE7c4akhro04a9LrzZorZ/QWo/XWo9PSUnpyl0LIUSP155APwD0bzSdEZjX4jpKKQvGZfiSjiigEEKI9mlPoG8EhiqlBiqlbMBVwBvN1nkDWBAY/x6wRurPhRCia7VZhx6oE78ZeA/jtsWntdbblVJ/BDZprd8A/g08o5TKBUoxQl8IIUQXatd96FrrVcCqZvPubjTuBOZ2bNGEEEKcCPkJlhBC9BAS6EII0UNIoAshRA8Rsk6ilVJFwL6TfHoyzX60FGakfKdGynfqwr2MUr6TN0Br3eIPeUIW6KdCKbWptV6vw4GU79RI+U5duJdRytc5pMpFCCF6CAl0IYToIbproD8R6gK0Qcp3aqR8py7cyyjl6wTdsg5dCCHEsbrrGboQQohmJNCFEKKHCOtAV0pdrJTapZTKVUrd0cJyu1LqxcDyz5VSWV1Ytv5KqbVKqWyl1Hal1G0trDNdKVWhlNoceNzd0rY6sYz5SqmtgX1vamG5Uko9Ejh+W5RSZ3Zh2YY3Oi6blVKVSqmfNluny4+fUupppdRRpdS2RvMSlVLvK6V2B4Ytds6mlFoQWGe3UmpBS+t0QtkeUkrtDPz9XlVKxbfy3OO+Fzq5jPcopQ40+jte0spzj/v/3onle7FR2fKVUptbeW6XHMNTorUOywdGy455wCDABnwDjGq2zo+BxwPjVwEvdmH5+gJnBsZjgJwWyjcdeCuExzAfSD7O8kuAdwAFTAI+D+Hf+jDGDyZCevyAqcCZwLZG8x4E7giM3wH8uYXnJQJ7AsOEwHhCF5TtQsASGP9zS2Vrz3uhk8t4D/CLdrwHjvv/3lnla7b8L8DdoTyGp/II5zP0YF+mWms3UN+XaWOzgeWB8ZeBmUop1RWF01of0lp/FRivAnZgdMXXncwGVmjDZ0C8UqpvCMoxE8jTWp/sL4c7jNZ6PUYT0I01fp8tB77TwlMvAt7XWpdqrcuA94GLO7tsWuv/aa29gcnPMDqgCZlWjl97tOf//ZQdr3yB7LgS+E9H77erhHOgt9SXafPAbNKXKVDfl2mXClT1jAM+b2HxOUqpb5RS7yilRndpwUAD/1NKfRnoz7W59hzjrnAVrf8ThfL41UvTWh8KjB8G0lpYJxyO5SKMb1wtaeu90NluDlQLPd1KlVU4HL8pwBGt9e5Wlof6GLYpnAO9W1BKRQOvAD/VWlc2W/wVRjXCWOD/gNe6uHjnaa3PBL4F/EQpNbWL99+mQC9YlwMvtbA41MfvGNr47h129/oqpX4LeIHnWlkllO+Fx4DBwBnAIYxqjXA0n+OfnYf9/1M4B3rY92WqlLJihPlzWuv/Nl+uta7UWlcHxlcBVqVUcleVT2t9IDA8CryK8bW2sfYc4872LeArrfWR5gtCffwaOVJfFRUYHm1hnZAdS6XU9cClwDWBD5xjtOO90Gm01ke01j6ttR94spV9h/S9GMiP7wIvtrZOKI9he4VzoId1X6aB+rZ/Azu01n9tZZ0+9XX6SqmzMY53l3zgKKWilFIx9eMYF8+2NVvtDeD7gbtdJgEVjaoW6sNPPwAAAUBJREFUukqrZ0WhPH7NNH6fLQBeb2Gd94ALlVIJgSqFCwPzOpVS6mLgV8DlWuvaVtZpz3uhM8vY+LrMnFb23Z7/9850AbBTa13Y0sJQH8N2C/VV2eM9MO7CyMG4+v3bwLw/Yrx5ARwYX9VzgS+AQV1YtvMwvnpvATYHHpcAPwR+GFjnZmA7xhX7z4DJXVi+QYH9fhMoQ/3xa1w+BTwaOL5bgfFd/PeNwgjouEbzQnr8MD5cDgEejHrcGzCuy3wA7AZWA4mBdccDTzV67qLAezEXWNhFZcvFqHuufw/W3/XVD1h1vPdCFx6/ZwLvry0YId23eRkD08f8v3dF+QLzl9W/7xqtG5JjeCoP+em/EEL0EOFc5SKEEOIESKALIUQPIYEuhBA9hAS6EEL0EBLoQgjRQ0igCyFEDyGBLoQQPcT/B/+Un4jueGN7AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Plotting\n", "pd.DataFrame(history.history).plot(lw=2);" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "from tensorflow.keras import models\n", "from tensorflow.keras import layers \n", "\n", "model = models.Sequential()\n", "model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "model.compile(optimizer='RMSprop',\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Retrain the model, this time using early stopping to stop training at the optimal time" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25000/25000 [==============================] - 2s 100us/sample - loss: 0.2879 - accuracy: 0.8862\n", "Loss: 0.2879, Accuracy: 0.8862\n" ] } ], "source": [ "# Based on the figure, we should stop after 4 epochs\n", "model.fit(x_train, y_train, epochs=4, batch_size=512, verbose=0)\n", "result = model.evaluate(x_test, y_test)\n", "print(\"Loss: {:.4f}, Accuracy: {:.4f}\".format(*result))" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Try to manually improve the score and explain what you observe. E.g. you could:\n", " - Try 3 hidden layers\n", " - Change to a higher learning rate (e.g. 0.4)\n", " - Try another optimizer (e.g. Adagrad)\n", " - Use more or fewer hidden units (e.g. 64)\n", " - `tanh` activation instead of `ReLU`" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3xW5f3/8dd17+xNFiuMMAMoQUDcFhmiqIBo3Sh+a+uoba1bqbVLbWv7q7VSq4gLBUQpRakIFgcqAdmEHSBhZC+Se1+/P86dmxACBEhyJ+HzfDzux5n3OZ/7Drxzcp1zrqO01gghhGj/TKEuQAghRPOQQBdCiA5CAl0IIToICXQhhOggJNCFEKKDsIRqx4mJibp79+6h2r0QQrRLq1evLtZaJzW2LGSB3r17d3JyckK1eyGEaJeUUnuOt0yaXIQQooOQQBdCiA5CAl0IIToICXQhhOggJNCFEKKDOGmgK6VeU0oVKqU2Hme5Ukr9VSm1Qym1Xil1bvOXKYQQ4mSacoQ+Cxh7guXjgN6B193Ay2delhBCiFN10uvQtdYrlFLdT7DKRGC2Nvrh/UYpFauUStVaH2imGoXoODy1UFMKZhtYHWANB5M51FWB1wWuanBXgftwI+OBl6va+AwWO9giwRZxnFdgmTXcGLfYGt+v1uCpMbbrqjL26apqZLrePM9hsAS+u/r7Co4fZ741HKxhoFRgv7WBz1R15LMdNV3VyLxqo16TGSxhxs+w4dAadvxldcPwRIjt0uw/xua4sSgd2FdvOj8w75hAV0rdjXEUT9euXZth10K0AVobIV19CKoPQnUhVAWGDaddFce+32wL/Edv+AoEUF141c232EH7we8Dv/fIy+c9evqYV2B9nxvcNfWC6jD4PS37HZms9YLVAR7nkcDW/pbd91GU8V16naB9rbjfBvpPhOtnN/tmW/VOUa31TGAmQHZ2tjxZQ7QMTy3UlkFtOTjLjx26AiFy1MNdAuPBeQ2n683ze6G6KBDgh4ygbmogmqwQnmBsw1NrHO353MarsbBvLSaLEbb2qCNHtvbIevMCYWyPNH75+FzGL4K6Xwzuw/VegaPYunFXtfH9OAPff0OWMGO7dfu2RzeYjjryqjsC93ka2W+DGurGPfXme53GNIDZ3uAzRh493di8uv37vcYvJa/T+DmecFgTWLf2yDAuo0V+jM0R6AVA/b8dOgfmCXEsd43xn9rrNP5T1oVZ3bjXfey8hi+v6/hhXVtuhE1rc8RAZPKRV1QKRHaCyMAwKsWYHxZn/MlfR2vjM3lqAgFf/xWY52047QRlNkK47mW2HD1tMhu/POpPm+tNB5sk6kLa3nLfTd1nDAZsjfGXRl1Amq0tt++GfN4j+2/N/baS5gj0hcC9Sqk5wHCgQtrPzxJaG/9Ba4rhcElgWASHixvMqzddd3TUksw2cMRCWGxgGFdvPNY4Agy2W9cL12DQqgbTHL2OMkFEUr0A72QExOlQyghTi92osyOq/xnD40Nbi9kC5ujQ1tCCThroSql3gUuARKVUPvA0YAXQWv8DWAyMB3YANcAdLVWsaCV+nxHCVQeMtt+6YfXBwPTBIyHtdZ7ats02CIs32lHNtsDL2mDc3si8euMWu3FE3FhYO2KPnPgS4izTlKtcbjzJcg38pNkqEi2rthwq8o8O6mOC+1DTTxhZHMbRangCRCQaZ+8jEgPTSfXmJRhDe5SErRAtJGTd54oWUlsO5XuP/2rqibfwBIhKNdp+o1LqjacazQx1QW2LkIAWoo2QQG9vnJVQtvs4gb3v5IFtDYeYLhCdenRI1x9GJrfsSTIhRIuQQG+LXNVQugtKd0LJTmO8ZKcxfbjoxO+1RkBs1+O8uhknpeSIWogOSQI9VNw1xpF2XVDXD+7qg8d/n8VhXMMa163xwG54WZwQ4qwhgd4avC7Yvxb2fQN7v4UDa6HyBJfqm21GaCf0hPgegWFPYxiVBibpJFMIcSwJ9JZwuBj2fQt7vzGG+783bqyoz2SBuO5Hgrp+cMd0bhv9ewgh2hUJ9DPl90PxNiO460K8dOex6yX1g67Doctw6DzMOAI3y9cvhGg+kiinyuepd/T9nTHesH8KSxh0zjbCu+sIY7yj3gUohGgzJNCb6nAxrH4dVv3LuAGnvqjUI+HdZTikZHXIfiKEEG2bBPrJHFgH374CG+Yd6fQpvif0vBS6jDCaUWK6yJUlQoiQk0BvjM8LuYuMIN/7dWCmgsyxMPz/oMelEuBCiDZHAr2+mlJY8wZ89ypU5hvz7NFwzs0w7C7jKhQhhGijJNABDm2Cb/8B698/0ntgQi847/9gyI1Gh1JCCNHGnb2B7vfB1o+NIM/74sj8XqNh+I+g52VyA48Qol05+wLdfRhyXoPvZhodWoHx1JQhP4Tz7obE3qGtTwghTtPZFehaw9zbYft/jem47kazyjk3GQ9MEEKIduysCnTvt+9T9sFKDhclQ1xPiIiD775GqZXGVSsmEyiFMgUeM6YUmJQxLzhtwmS3Y+/dC3ufvjj69cWSkoJqoate/C4XrtxcajduxLlxE86NG3Hv2QMmE8piQZnNYLEcGbdaUGZjGos5OH7UMrsdS0IClqQkLJ06HT1MiDfe20ZprdEuF8pqNT6TECKo7f7PbUau3bsp/de/qFgwH+0LnOAszAPymmX7ppgYHH36YO/bB0fffjj69sHWqxcmm+2UtqPdbpzbtuPcuBHnpo3UbtyEa/t28HobX785im/IZMKcEI8lKQlrUicsnZKODvykJCwJCajwcEwOB8puRzXDuQbtduMtKcFbVIS3uBhvUTHeYmPcF5w2XtppnLhWVivK4UA57JjsgaEjrN60A5PdbgwdgWmHHWV3oMwmMJmDQ0zK+AVhMgWG5qPXUSYwG8uU1Ya9Zw8sqakt9otciNOhjCfItb7s7Gydk5PTovuoWfM9Ja/9i+rPlhnNLUBkhoW4X/4FU0wcaD9ojfb7wa8BDX4/Wmtjuv5yDWg/2u/HX1WNa9s2nLm5uHJz8ZWXH7tziwV7Rgb2fn1x9OkbCPu+WBISANAeD66dO3Fu3Bg8+nZt3Yr2eI7ejsmEvWcPHAMG4hg4kLCBA7D16o0yKbTXi/b5jPf4fEePe71orw983uC49hrL/LVOvCXFeAsLjQAtLDKGRUX4SkpO+XtWNtuR8AwLOxKigWHDoMVswldaFgjoInxFxfgqmvgkJYwg115v8GcaKuaYGOz9+uHo1w9Hf2Noy8iQvxxEi1JKrdZaZze6rKMFuvb7qV62jJJ/vUbt998DRgDEdKsmPrMC+70fQI+Lm29/WuMtLMSVm4tzSy7Orbm4crfizstrNHDMSYlYkzrh2rkT7XIds9yWkREMbsfAgTj69sUUEdFs9Z6M9niOHCkfFfiFeAuL8BQV4ispxe90op3ORj/DaTGbjzQDJSZiTkrEkpiIJdGYtgSnEzFFRBhNL2432unE73ShXc5gTf5AXca0C7+zFh1cxxhqn9/4xef3G7/E/T7wHRka8+qtU29df00Nrq1bG/1FrhwO7JmZRsgHgt6emYnJ4Wie76kJtNeL5+AhPAUFePLzjWFBAe6CfDwF+/EeOmT8JWKzYbJawWbFZLUZf/HYbMarsfF6QyBw8BM4GPL7Awc82vh332DZ0etpMJsDzYVmlMV6ZNxsOXrcEmhSrBs3m1EmM1DvQEz70T7fkXG/3/gZ1tVT9/OtV4uyWo0DDpsdZbdjstuMvzZtdpTdZiwLTJvsge+hbp7VBorAn8iBz1v3fz0wrhsuC87XoMEUEYE1udNp/XzPikD3u1xUfPgRpa+/boQpRlNI3A03EB+7GsuehdDvapj6ZrPt84T11Nbi2r4d55ZcXFtzceZuxbV1K/7Dh4PrWLt1JSxw5O0YOABH//6YIyNbpb7mUtemrZ1O/C4XurbWGDYMWpcLf20gWH0+LPFxmOsCOykRc2xsszTdtBatNd5Dh3Bu3oJzy2acW7bg2rwFz/79x65sNmPvkYG9rxHy9j6ZmMLDj5zbMFtQ1nrjFvORcyPmo8cxm8Hvx1tYaIR0MLD3HwnvgwfB18SHfIuQiB4/jvQ//em03tuhA91bVkbZu+9S9vY7weYCa1oa8bffRuykSZiK18Hr44wn/fzkO+NJPyGi/X48BQV4Cwux9+qFOUaurOlofOXlOHO34txiBL1ryxZcu3Y3b8AqddLmJkunTlg7d8aano41PQ1rejq2wLQlJcU4wPR48LvdaLcH7fEYf/F46qbdgenA/PrjHo9RgzIZ5x5MpiPjwYsLTMbFBfWXBS46AGUcPddrBtReH9rnDY4bTYWBpkOfF7xGk2LdfJQ6cm7DVLd989H1mE1H12YyB+oxPrt2BT6X22UchLjcxsGJ24W/btzlwu+ut8zlwu8OPNtAgUIFvgt1pDsQpRpfVm9+5CUXk/zoo6f54z9+oLfbk6Lu/HxKX59F+QcfoGtrAXD070/8ndOIHjPGuFLD74PFvzTeMOqnIQ1zAGUyYevSBVuXLiGtQ7Qcc2wsESOGEzFieHCe3+k0/loLHM27d+4ygiRw3oO6cyF1wRWYPjL/yDg+H2iNOSkRW1p6vdBOD4R2Opa0tCadkFc2W6s254mW1+4CvXbDRkpe+xdVS/5rtMcBERdeSMKd0wgfPvzoqw5Wz4JDG4zeEEc9EJqCxVnP5HAQlpVFWFbWGW9LB9qn5cSraEy7CnStNQcefxzXtm1gsRBz1VXET5uGo0/msSvXlMKyXxvjVzwLtvDWLVaIFqCUMtrRhWhEuwp0pRSJ9/yI2g0bib/1FqwpKcdfeflvobYMul8I/Se2XpFCCBEi7SrQAaLHjSN63LgTr3RwI+T8C5QZxv1B+i4XQpwV2s91Yk2lNXz8sHFT0LC7IHlAqCsSQohW0fECfdMC2PMlhMXDpad3WZAQQrRHHSvQ3TXw3yeN8cufgrC40NYjhBCtqGMF+lcvGo+OSxkE594a6mqEEKJVNSnQlVJjlVJblVI7lFKPNLK8q1JquVLqe6XUeqXU+OYv9STK8uDLF43x8c8bPegJIcRZ5KSBrpQyAy8B44D+wI1Kqf4NVnsCeF9rfQ5wA/D35i70pP77BPhckHU9dB3R6rsXQohQa8oR+nnADq31Lq21G5gDNLywWwPRgfEYoJEeilrQzuWw5d9gjYDRv2rVXQshRFvRlEBPB/bVm84PzKtvBnCzUiofWAzc19iGlFJ3K6VylFI5RUVFp1FuI3we+CTQCnTRLyA6rXm2K4QQ7UxznRS9EZilte4MjAfeVEods22t9UytdbbWOjspKal59rzqVSjKhbgMGPmT5tmmEEK0Q00J9AKgfveAnQPz6rsTeB9Aa70ScACJzVHgCVUXwfLfGeNjfwcWe4vvUggh2qqmBPoqoLdSKkMpZcM46bmwwTp7gcsBlFL9MAK9mdpUTmDZM+CqgF4/gMyxLb47IYRoy04a6FprL3AvsATYgnE1yyal1DNKqasDq/0cmK6UWge8C9yuW/rJGQVrYM2bYLLA2N9Lfy1CiLNekzrn0lovxjjZWX/eU/XGNwOjmre0E/D7jf5a0DDiHkjs3Wq7FkKItqp93im64X3I/w4iOsFFvwx1NUII0Sa0v0B3VcGngT8ORv8KHNEnXl8IIdoQt89NjaemRbbd/gJ9xfNQfQjSs2HQDaGuRgghmqTcWc7M9TMZM38Mb215q0X20b4ecOH3w/7vjfHxzxlP8BZCiDZsb+Ve3tz8Jh/t/Ihar/FA+9WHVrfIvtpXoJtMcOtCyM+B9KGhrkYIIY5rbeFaZm2axbK9y9AYF/2NShvFbQNuY0Rqy/Q31b4CHYzLE7sMC3UVQghxDJ/fx2d7P+ONzW+wvmg9AFaTlSt7XMmt/W+ld1zLXpHX/gJdCCHamBpPDQt2LODNzW9SUG3cSB9ti2Zqn6nc2PdGksKbqauTk5BAF0KI01RYU8g7W97h/W3vU+WuAqBzZGduHXArE3tOJNwa3qr1SKALIcQp2lq6ldmbZ7N492K8fi8AQ5KGcNuA27i0y6WYQ/SAHQl0IYRoAq01K/ev5I3Nb/D1/q8BMCkTo7uN5tb+tzKk05AQVyiBLoQQJ+Txefg472NmbZrF9rLtAIRZwrim1zXc0u8WukR3OckWWo8EuhBCNKLSXcncrXN5Z8s7FNYWApAYlshN/W5iSuYUYuwxIa7wWBLoQghRT0F1AW9tfosPtn9Ajde4Rb9XbC9uG3Ab4zPGYzPbQlzh8UmgCyEEsKl4E7M2zeLTPZ/i0z4AhqcO5/YBtzMqbRSqHXTRLYEuhDhr+bWfL/K/YNamWeQcygHAoixM6DGB2wbcRt/4viGu8NRIoAshzjoun4t/7/w3szfPZnfFbgAirZFMzpzMTf1uIiUiJcQVnh4JdCHEWaPSXck7W97h3dx3KXWWApASkcLN/W5mUu9JRNoiQ1zhmZFAF0J0eJXuSt7a/BZvbX6LKo9xR2e/+H7cNuA2ruh+BVaTNcQVNg8JdCFEh9VYkA9PHc70rOmcl3JeuzjReSok0IUQHU6lu5K3N7/Nm5vfPCrI7xl8D0OTO27X2xLoQogO42wN8jpKax2SHWdnZ+ucnJyQ7FsIcSyPx0N+fj5OpzPUpZwyv/Zz2HOYw57D+LUfALvZTqQtErvZHuLqTo/D4aBz585YrUe37yulVmutsxt7jxyhCyEAyM/PJyoqiu7du7ebtmWf30eJs4SS2hIidAQRRBBhjSApPIkIa0SoyzttWmtKSkrIz88nIyOjye+TQBdCAOB0OttNmNcP8roj8o4Q5HWUUiQkJFBUVHRK75NAF0IEtfUw9/q9lDpLO2yQ13c6PwsJdCFEm+f0Oil1llLuKqfuvF+4NZxO4Z06XJCfCQl0IUSbERkZSXV1NWC0I1e5qyhxllDjqTmyji2SREciETYJ8oYk0IUQbYrX76XMWUaZswyP3wMYTwaKtccS74jHbmmfV620Bgl0IcQxuj/ynxbZbt7vrzzuMqfXiUazrWwbfr+fP/7qj3z12VdYzBaefPxJbrzxRg4cOMDUqVOprKzE6/Xy8ssvc/7553PnnXeSk5ODUopp06bx4IMPtkj9bZ0EuhAiZBo2q2it0Vrz5SdfsmfLHjZv2ExJSQnDhg3jkksu4Z133mHMmDE8/vjj+Hw+ampqWLt2LQUFBWzcuBGA8vLyEH+q0GlSoCulxgJ/AczAq1rr3zeyzvXADEAD67TWP2zGOoUQrehER9LN4XjNKkopesX24tU1r3LzTTdjsVhITk7m4osvZtWqVQwbNoxp06bh8Xi45pprGDJkCD169GDXrl3cd999XHnllVxxxRUtWntbZjrZCkopM/ASMA7oD9yolOrfYJ3ewKPAKK31AOCnLVCrEKKdc3qd7K/ez7aybRTWFOLxe7CZbaREpJAZl4lCnbCN/KKLLmLFihWkp6dz++23M3v2bOLi4li3bh2XXHIJ//jHP7jrrrta8RO1LScNdOA8YIfWepfW2g3MASY2WGc68JLWugxAa13YvGUKIdorrTXV7mr2VO5hZ/lOypxlaK2JtEXSNborvWJ7kRCWgNlkDr7nwgsv5L333sPn81FUVMSKFSs477zz2LNnD8nJyUyfPp277rqLNWvWUFxcjN/vZ9KkSTz77LOsWbMmhJ82tJrS5JIO7Ks3nQ8Mb7BOJoBS6iuMZpkZWutPmqVCIUS75Nd+KlwVlNSW4PK5AKNZJcYeQ4Ij4YRH4tdeey0rV65k8ODBKKV47rnnSElJ4Y033uD555/HarUSGRnJ7NmzKSgo4I477sDvN240+t3vftcqn68tOmnnXEqpycBYrfVdgelbgOFa63vrrbMI8ADXA52BFUCW1rq8wbbuBu4G6Nq169A9e/Y040cRQpyJLVu20K9fvzPejsfvocxZRqmzFJ/feNiyxWQh3hFPnCMOi0muxWiqxn4mZ9o5VwHQpd5058C8+vKBb7XWHmC3Umob0BtYVX8lrfVMYCYYvS02Yd+N8vs1JlPbvkVZiLON0+ukxFlChasieDenw+IgISyBaFs0JtWUFl5xJpryDa8CeiulMpRSNuAGYGGDdT4ELgFQSiViNMHsasY6g15YspVRf1hGYWX76+JTiI6m7rLDvIo8dpbvpNxp3JofZYuie0x3esT0INYeK2HeSk56hK619iql7gWWYLSPv6a13qSUegbI0VovDCy7Qim1GfABD2mtS1qi4O2FVRyocPLB9wX86OKeLbELIcRJ+LWfclc5pbWlR7WPx9pjiQ+Lb7d9kLd3TWrM0lovBhY3mPdUvXEN/CzwalFThnZhyaZDzM3Zx/9d1KPN9w4nREfh9Xtxep0c9hymzFUm7eNtULv79i/uk0RipI2dRYdZu6+cc7rGhbokIToUrTVuvxun12m8fMbQ6/cetZ60j7c97S7QrWYT156Tzj+/2M3c1fkS6EKcgRpPDTvKd5BbmktXV1d2VezC5XUF+xqvz6RM2C12HGYHMfYYwi3h8hdyG9PuAh1g8tAu/POL3fx73X6emtAfh9V88jcJcZarcFWwqWQTm0s2k1uay9bSreyp3IPGuCLlxf4vUuupBYxmFIfFgcPsCA5tZpsEeBvXLgO9T0oUgzrHsD6/giWbDjJxSHqoSxKiTXF6neSW5rKheAMbizeyqWQTeyqPve/DoixkxGbQN64v0fZoukV3w2FxdPi2cK/Xi8XS8T5ju/1EU4Z2Zn1+BfNW50ugi7Oax+9hZ/lONhZvDL52lO/Ap31HrWcz2eib0JeBCQPpl9CPPnF96BnbE5vZBhg3sUTaIo2VZ8S0TLEzKk66yjXXXMO+fftwOp088MAD3H333XzyySc89thj+Hw+EhMT+eyzz6iurua+++4Ldpv79NNPM2nSpKMekjFv3jwWLVrErFmzuP3223E4HHz//feMGjWKG264gQceeACn00lYWBivv/46ffr0wefz8fDDD/PJJ59gMpmYPn06AwYM4K9//SsffvghAJ9++il///vfWbBgQct8T6ep3Qb6VYPT+PWiLXy5o5j95bWkxYaFuiQhWsXeyr2sL17PpuJNbCzeyJbSLcFLB+uYlInMuEwGJg5kQMIAshKz6BXXC6vJGqKqm+61114jPj6e2tpahg0bxsSJE5k+fTorVqwgIyOD0tJSAH79618TExPDhg0bACgrKzvptvPz8/n6668xm81UVlbyxRdfYLFYWLp0KY899hjz589n5syZ5OXlsXbtWiwWC6WlpcTFxfHjH/+YoqIikpKSeP3115k2bVqLfg+no90Gemy4jdEDkvnP+gN8sCafey/rHeqShGgxhz2HWbx7MfO2zWNzyeZjlneJ6sLAxIEMTBjIwMSB9I3vS7g1/PR32IQj6Zby17/+NXjku2/fPmbOnMlFF11ERkYGAPHx8QAsXbqUOXPmBN8XF3fyCySmTJmC2Wycc6uoqOC2225j+/btKKXweDzB7f7oRz8KNsnU7e+WW27hrbfe4o477mDlypXMnj27mT5x82m3gQ4weWhn/rP+APNW5/OTS3vJCRvR4Wwq3sTcbXNZvHsxtV7jhGWULYqhyUMZmDCQrMQsBiQOIMbeQk0krezzzz9n6dKlrFy5kvDwcC655BKGDBlCbm5uk7dRPweczqPvKI+IOPIc0ieffJJLL72UBQsWkJeXxyWXXHLC7d5xxx1cddVVOBwOpkyZ0ibb4NteRafgot5JJEfbySupIWdPGcO6x4e6JCHOWLW7Ong0vqV0S3D+0OShTM6czOhuozvsnZgVFRXExcURHh5Obm4u33zzDU6nkxUrVrB79+5gk0t8fDyjR4/mpZde4sUXXwSMJpe4uDiSk5PZsmULffr0YcGCBURFRR13X+npxvm3WbNmBeePHj2aV155hUsvvTTY5BIfH09aWhppaWk8++yzLF26tMW/i9PRru8GMJsU157TGYB5OfkhrkaI06e1ZmPxRmZ8PYPL5l7Gr7/5NVtKtxBjj+GW/rfw0cSPmDV2FhN6TOiwYQ4wduxYvF4v/fr145FHHmHEiBEkJSUxc+ZMrrvuOgYPHszUqVMBeOKJJygrK2PgwIEMHjyY5cuXA/D73/+eCRMmcP7555Oamnrcff3yl7/k0Ucf5ZxzzsHrPXLT1F133UXXrl0ZNGgQgwcP5p133gkuu+mmm+jSpUuz9ErZEk7afW5Lyc7O1jk5OWe8nZ1F1Vz+x/8RYTOz6okfEG5r1390iLNMlbuKxbsWM2/7PHJLjzQrZCdnMzlzMj/o9oNWC/Dm6j63I7v33ns555xzuPPOO1tlfy3RfW6b1jMpknO7xrJmbzkfbzjIpKGdQ12SECdUdzQ+b/s8Pt79cbBtPNYey8SeE7ku8zp6xPQIcZWioaFDhxIREcEf//jHUJdyXO0+0MG4c3TN3nLmrc6XQBdt2ncHvuO5Vc+xtWxrcN6wlGFM7m0cjdddEy7antWrV4e6hJPqEIE+YXAqv/r3JlbuKmFfaQ1d4s/gci0hWoDH5+H/rf1/zNo4C40mzh7HxF4Tua73dWTEZIS6PNFBdIhAj3ZYGTswhY/W7mf+mnx++oPMUJckRFBeRR4Pf/Ewm0s2Y1Im7hl0D3dm3SlH46LZteurXOqbMtR4St681fn4/aE50StEfVprPtj+Adcvup7NJZtJj0znjbFvcM+QeyTMRYvoEEfoACN7JpAW4yC/rJZvd5cysmdCqEsSZ7EKVwW/WvkrPt3zKQDjM8bzxIgniLI1fk20EM2hwxyhm00qeEJ07up9Ia5GnM1WHVzFpIWT+HTPp0RYI/jtBb/lDxf9QcK8mUVGRh53WV5eHgMHDmzFatqGDhPoYHQFAPDxhoNUu7wnWVuI5uXxe/jLmr9w55I7OVRziEFJg5h71Vyu6nlVqEsTZ4kO0+QC0C0hgvO6x/NdXimL1x/g+mFdQl2SOEvsrdzLwyseZmPJRkzKxN1Zd/N/g/+vXfRu2JisN7JaZLsbbttw3GWPPPIIXbp04Sc/+QkAM2bMwGKxsHz5csrKyvB4PDz77LNMnDjxlPbpdDq55557yMnJwWKx8Kc//YlLL72UTZs2cccdd+B2u/H7/cyfP5+0tB5ieyMAACAASURBVDSuv/568vPz8fl8PPnkk8E7U9uDDhXoAJOzO/NdXilzV++TQBctTmvNRzs/4nff/o4abw2pEan87sLfMTR5aKhLa3emTp3KT3/602Cgv//++yxZsoT777+f6OhoiouLGTFiBFdfffUpdcT30ksvoZRiw4YN5ObmcsUVV7Bt2zb+8Y9/8MADD3DTTTfhdrvx+XwsXryYtLQ0/vOf/wBGfy/tSbsM9ApXxXF7lxuflcrTH21iVV4ZecWH6Z4Y0eh6QpypSnclz6x8hiV5SwAY230sT458kmhbdIgrO3MnOpJuKeeccw6FhYXs37+foqIi4uLiSElJ4cEHH2TFihWYTCYKCgo4dOgQKSkpTd7ul19+yX333QdA37596datG9u2bWPkyJH85je/IT8/n+uuu47evXuTlZXFz3/+cx5++GEmTJjAhRde2FIft0W0qzZ0n9/HH3P+yLj549hfvb/RdSLtFsZnGR3yzFstHXaJlrH60GomL5zMkrwlhFvCeXbUszx30XMdIsxDacqUKcybN4/33nuPqVOn8vbbb1NUVMTq1atZu3YtycnJx3SJe7p++MMfsnDhQsLCwhg/fjzLli0jMzOTNWvWkJWVxRNPPMEzzzzTLPtqLe0q0M0mMwXVBVR5qnj666c5XsdiU7KNk6Pz1+Tjk2vSRTNy+Vz87fu/MW3JNA4cPsDAhIHMvWouE3tNlP74m8HUqVOZM2cO8+bNY8qUKVRUVNCpUyesVivLly9nz55jn4t6MhdeeCFvv/02ANu2bWPv3r306dOHXbt20aNHD+6//34mTpzI+vXr2b9/P+Hh4dx888089NBDrFmzprk/Yotqd00ujw9/nJyDOXxz4BvmbZ/HlMwpx6xzXvd4usSHsa+0lq93FnNh76QQVCo6kl3lu5i3fR4Ldy6kwlWBQjE9azr3DLmn3Z74bIsGDBhAVVUV6enppKamctNNN3HVVVeRlZVFdnY2ffv2PeVt/vjHP+aee+4hKysLi8XCrFmzsNvtvP/++7z55ptYrVZSUlJ47LHHWLVqFQ899BAmkwmr1crLL7/cAp+y5bTL7nM/yfuEh/73EOGWcBZMXEBaZNox6/xl6Xb+vHQbVw9O4683nnOm5YqzkNPr5NM9nzJv2zzWFB45UusX34+Hhj3EsJRhIayu+Un3uW3PWdF97phuY/hvt//y6Z5PmfH1DF4Z/coxf+5ed246f166jSWbDlJR6yEmTI6iRNPsKNvB/O3zWbhzIZXuSgDCLeGM7zGeyZmTGZAwIMQVCtG4dhnoSikeG/4Yqw6uYuWBlczfPp/JmZOPWqdLfDjn90zg650lLFq/n5uGdwtRtaI9qPXWBo/Gvy/8Pjh/QMIAJmdOZlzGOCKscsVUW7NhwwZuueWWo+bZ7Xa+/fbbEFUUWu0y0AESwxJ5fPjjPLTiIV7IeYFRaaNIjTz6cVOTh3bm650lzFudL4EuGrWtbBvzts1j0c5FVHmqAIiwRnBlxpVMypxE/4T+Ia5QnEhWVhZr164NdRltRrsNdIAx3cewJG8JS/cu5emvnz6m6WXcwFSe+mgT3+8tZ0dhFb06SV8aAmo8NSzJW8K87fNYX7Q+OD8rMYvJmZMZ230s4VbpU1+0P+060JVSPD7icVYdMppePtj+AZMyJwWXh9nMTBiUypxV+5i7Op9Hx8kJn1Cr8dSwo3wHmXGZOCyOVt33rvJdvLf1PRbuXEi1pxqASGskV/a4ksmZk+kbf+pXUAjRljQp0JVSY4G/AGbgVa3174+z3iRgHjBMa33mT4BugsSwRB477zEe/uJhns95nvPTzj+q6WXy0M7MWbWPBWsKeOiKPljM7erS+w7D5/excOdC/vb93yisLSTKGsW4jHFc2/taBiQMaLFruL1+L5/v+5w5uXP49uCRdtVBSYOY3HsyY7qPkaNx0WGcNNCVUmbgJWA0kA+sUkot1FpvbrBeFPAA0OpnI8ZljOO/e/7LZ3s/41crf8XLP3g5GBBDu8WRkRjB7uLDfLG9mEv7dmrt8s563x74lhdyXgg+1T7BkUCJs4T3t73P+9vep1dsL67pdQ0TekwgIax5+rEvri1m/rb5zN02l0M1hwAIs4QxPmM8N/a9kT7xfZplP0K0JU05Qj8P2KG13gWglJoDTAQ2N1jv18AfgIeatcImUErxxIgnyDmUw1f7v2LBjgVc1/u64LLJQzvz/JKtzFudL4HeinZX7OZPq//E5/s+ByA5PJmfDv0p4zPGs6N8Bx/u+JD/7PoPO8p38ELOC7y4+kUu6nwR1/a+lgvSL8BiOrUWQa01a4vW8m7uu3y651O8fqML5e7R3ZnaZypX97pabs3vQCIjI6murg51GW1KU/7HpAP1nxiRDwyvv4JS6lygi9b6P0qp4wa6Uupu4G6Arl27nnq1J5AYlsij5z3KI188wvOrjKaXlAijA5/rzk3nhf9u5dPNhyivcRMbLo//akllzjL+se4fvL/1fbzaS7glnDuz7uSW/rcQZgkDIDMuk18O+yUPnvsgK/JXsGDHAr4s+JJl+5axbN8yEsMSuarHVVzT6xp6xPY44f5qPDUs3r2YOblz2Fq2FQCTMnFpl0u5oe8NjEgdgUlJU5toGV6vF4ulbZyOPOMqlFIm4E/A7SdbV2s9E5gJxp2iZ7rvhsZnjGdJ3hKW71vOjK9nBJteUmPCuKBXIl9sL+ajtfu57fzuzb1rAbh9bt7NfZdX1r1ClacKkzIxqfck7j3nXhLDEht9j9Vs5fJul3N5t8spqini37v+zYLtC8irzOP1Ta/z+qbXGZw0mGt6XcPY7mOJtB15Sk1eRR7vbX2Pj3Z8FLzkMN4Rz3W9r2NK5pRG7yAWTbOlb8tcQNAvd8txlzVnf+jV1dVMnDix0ffNnj2bF154AaUUgwYN4s033+TQoUP86Ec/YteuXQC8/PLLpKWlMWHCBDZu3AjACy+8QHV1NTNmzOCSSy5hyJAhfPnll9x4441kZmby7LPP4na7SUhI4O233yY5OZnq6mruu+8+cnJyUErx9NNPU1FRwfr163nxxRcB+Oc//8nmzZv585//fEbfLTQt0AuA+h2Ldw7MqxMFDAQ+D7RbpwALlVJXt9aJ0TpKKZ4a+RSrD63mq/1f8eGOD7m297UATMnuwhfbi5m3Ol8CvZlprfl0z6f8efWfya82ergcmTqSXwz7BZlxmU3eTlJ4EtMGTuOOAXewrmgdH+74kI93f8y6onWsK1rHH777A1d0v4Ls5Gw+3v0xKw+sDL53UNIgbuhzA2O6j5EHMLdTzdkfusPhYMGCBce8b/PmzTz77LN8/fXXJCYmUlpaCsD999/PxRdfzIIFC/D5fFRXV1NWVnbCfbjdbuq6LykrK+Obb75BKcWrr77Kc889xx//+Ed+/etfExMTw4YNG4LrWa1WfvOb3/D8889jtVp5/fXXeeWVV8706wOaFuirgN5KqQyMIL8B+GHdQq11BRA8/FJKfQ78orXDvE5iWCKPDn+UR794lOdWPcfItJGkRKRwRf9kohwWNhRUkHuwkr4p0pbaHDYUbeD5nOeDd1f2jOnJz7N/zgXpF5z2lStKKYZ0GsKQTkP45bBfsnTvUhZsX0DOoRwW7lzIwp0LAbCb7YzPGM8NfW+QG4Ca2YmOpFtKc/aHrrXmscceO+Z9y5YtY8qUKSQmGpEVHx8PwLJly5g9ezYAZrOZmJiYkwZ6/ScZ5efnM3XqVA4cOIDb7SYjIwOApUuXMmfOnOB6cXFxAFx22WUsWrSIfv364fF4yMpqnidEnTTQtdZepdS9wBKMyxZf01pvUko9A+RorRc2SyXN6MqMK1mSt4TP933OjJUzePnyl3FYzVw9OI23v93LvJx8npggAXAmDlQf4MU1L7J492LAaOr4yZCfcF3v6075ZOaJhFvDubrn1Vzd82r2Ve7jw50fsqFoA6PSR3FNr2uO+6AT0T7V9Yd+8ODBY/pDt1qtdO/evUn9oZ/u++qzWCz4/f7gdMP3R0Qc6Qrivvvu42c/+xlXX301n3/+OTNmzDjhtu+66y5++9vf0rdvX+64445TqutEmnSmSGu9WGudqbXuqbX+TWDeU42Fudb6klAdnddRSvHUiKeItkXzVYHR9AJGswvAh2sL8Pj8J9qEOI4aTw1/XfNXrvrwKhbvXozNZGPawGksunYR1/e5vlnDvKEu0V2475z7mHnFTG4bcJuEeQfUXP2hH+99l112GXPnzqWkpAQg2ORy+eWXB7vK9fl8VFRUkJycTGFhISUlJbhcLhYtWnTC/aWnpwPwxhtvBOePHj2al156KThdd9Q/fPhw9u3bxzvvvMONN97Y1K/npDrsqf+k8CQeOe8RAJ5f9TwHDx9kcOcYenWKpLjaLU8zOg2bSjZx/aLr+eeGf+LyuRjXfRwLr13Ig0MfJMom3SqIM9dYf+g5OTlkZWUxe/bsJveHfrz3DRgwgMcff5yLL76YwYMH87Of/QyAv/zlLyxfvpysrCyGDh3K5s2bsVqtPPXUU5x33nmMHj36hPueMWMGU6ZMYejQocHmHIAnnniCsrIyBg4cyODBg1m+fHlw2fXXX8+oUaOCzTDNoV32h95UWmvuX3Y/n+d/zgXpF/D3y//OwnX7eWDOWiLtFpY8eBHpsWEtWkNHoLXmzc1v8uc1f8br99I7rjdPj3yawUmDQ12aaEbSH3rrmjBhAg8++CCXX375cdc51f7QO+wROhhNL0+OfJIoWxRfFnzJRzs/4urBaYwZkEy1y8vD89Yf9zF2wlDqLOUnn/2E53Oex+v3ckOfG3j3ynclzIU4TeXl5WRmZhIWFnbCMD8dbeNq+BbUKbwTj573KI99+RjPffccI1NH8ptrs1iVV8aXO4p5+9u93DxCutZtzLcHvuXRLx6lqLaIaFs0z4x6hsu7Nu8/QCHORHvsDz02NpZt27a1yLY79BF6nQk9JnBx54up8lTxq5W/IiHCxq8nDgTgt4u3sK+0ptVq8Ws/bp+71fZ3Ojx+D39Z8xem/3c6RbVFnNvpXOZfPV/C/CzQ3v5iresPvf6rLYf5qTidn8VZEeh1NxxF2aL4ouALFu5cyJWDUrlyUCo1bh+/mLsOv79l/yF7/V7+vfPfXPfRdQx9ayj3L7ufjcUbW3Sfp6OguoDbP7mdVze8ilKKHw/+Mf8a869gNwqi43I4HJSUlLS7UO+ItNaUlJTgcJxaF9Md+qRoQwt3LuTxLx8nyhrFz7J/xohOl3PN33IornYz46r+3D4qo9n36fK5+GjHR7y28TUKqguOWT4qbRR3D7qbc5PPbfZ9n6oleUv41de/ospTRXJ4Mr+/8PdkpzR67kV0QB6Ph/z8/FO+Xlu0DIfDQefOnbFaj34e8olOip5Vga615oHlD7B8n3HpUJgljKzYS1i+KgObrxufPHAx3ROb57mRNZ4a5m6byxub3qCotggwev2bNnAaI9NG8k7uO7yX+x41XqO5Jzs5m+mDpjMydWSL9Q1+PLXeWv7w3R+Yv30+AJd2uZRnzn+GWEdsq9YhhDg5CfR6PD4Pi3cv5oPtH7CmcE1wvs+ZQqrpYubefC/xYacfZBWuCt7Z8g5vbXkr+MT4vvF9uSvrLn7Q9QeYTeaj1n17y9u8teUtqtxG51JZiVncPehuLu58casE+7aybTz0v4fYVbELm8nGQ8MeYmqfqa3+S0UI0TQS6Mexq3wXH2z/gA93fESFuxwAs7IyNuMKJvWeRHZydpODraimiNmbZ/Pe1veo9dYCMCRpCNMHTefC9AtPuJ1qdzVzts5h9qbZlLmMO8n6xPVh+qDpx/wSaC5aa97b+h7Pr3oet99Nj5gePHfRc/LgByHaOAn0k3D73Py/bz7k1bVzsERuD87vGtWV63pfx8ReE4/b/Wt+VT6vb3ydD3d8iNtvXL0yKm0Ud2XdxdDkoad0pFvjqWH+9vnM2jiLwtpCADJiMpieNZ1xGeOa7bb6ClcFT3/9NJ/t/QyASb0n8cthv5RHsQnRDkigN9FDc9cxb/16OnfdiDVmVbDt26IsXNzlYib1nsT5aedjNpnZWb6TVze8yse7P8anfQD8oOsPuGvQXQxIGHBGdTR2IjU9Mp07s+5kYs+JJ+0e1uPzUO4qp8xVRrmzwdBVztI9SzlUc4goaxRPnf8UY7uPPaN6hRCtRwK9iSpqPYx9cQUHKpw8NKY3A3sdYP72+azIXxEM7ZSIFHrG9OSr/V8BYFZmruxxJdMGTqNnbM9mrcfj97B412Je3fAqeZV5gHGj1PWZ12M2mSlzGgHdcFj3RPsTGZQ0iOcueo70yPRmrVkI0bIk0E/B/7YVcdtr32Ezm/j3fRfQJyWKwppCFu5cyPxt84MPcLCZbFzb+1ruGHhHi4eiz+/j0z2f8sr6V9hRvuOk65uVmVh7LHGOuKOGdeOpEalc3OVirCbrSbclhGhbJNBP0aMfbODd7/aSlR7DBz8+H6vZuP/Kr/18d/A79lTs4fJulx+3Xb2l+LWfz/d9ztf7vybCGkGcPY5YR+wxwyhrlFylIkQHJYF+iqpdXsb8eQUF5bX8bHQm91/eO9QlCSEEcBb3tni6Iu0Wnp88CIC/fradTfsrQlyREEKcnAT6cZzfK5FbR3bD69f8Yu563F55wpEQom2TQD+Bh8f2pWt8OFsOVPK3ZdtP/gYhhAghCfQTiLBbeGHKYJSClz7fyYZ8aXoRQrRdEugncV5GPHecn4HPr/n53LW4vL5QlySEEI2SQG+Ch8b0oUdiBNsOVfPiUml6EUK0TRLoTRBmM/P8lMGYFLzyv518v7cs1CUJIcQxJNCbaGi3OKZf2AO/hp/PXcdhlzfUJQkhxFEk0E/Bg6Mz6dUpkl1Fh5n08tet+ixSIYQ4GQn0U+Cwmpl5y1AyEiPIPVjF1X/7kpU7S0JdlhBCABLop6xHUiQf/ngUF2UmUVbj4eZ/fcvslXnyYF0hRMhJoJ+GmHArr98+jP+7qAc+v+apjzbx2IINcjepECKkJNBPk9mkeHR8P16cOgS7xcS73+3jh//8hqIqV6hLE0KcpSTQz9A156Qz90cjSY1xkLOnjKv/9iXr88tDXZYQ4iwkgd4MBnWO5aN7RzG0WxwHKpxM+cdKPlpbEOqyhBBnmSYFulJqrFJqq1Jqh1LqkUaW/0wptVkptV4p9ZlSqlvzl9q2dYpy8M704UzN7oLL6+eBOWv53cdb8PnlZKkQonWcNNCVUmbgJWAc0B+4USnVv8Fq3wPZWutBwDzgueYutD2wW8z8flIWz0wcgNmkeOV/u5g2axUVtZ5QlyaEOAs05Qj9PGCH1nqX1toNzAEm1l9Ba71ca113l803QOfmLbP9UEpx68juvHXncOLCrfxvWxHXvPQVOwpP/uBmIYQ4E00J9HRgX73p/MC847kT+LixBUqpu5VSOUqpnKKioqZX2Q6N7JnAwnsvoG9KFLuLD3PtS1+xLPdQqMsSQnRgzXpSVCl1M5ANPN/Ycq31TK11ttY6OykpqTl33SZ1iQ/ngx+fz/isFKpcXu58I4e/f75DbkISQrSIpgR6AdCl3nTnwLyjKKV+ADwOXK21louxA8JtFl764bn84opMtIbnPtnKfe9+T2GlM9SlCSE6mKYE+iqgt1IqQyllA24AFtZfQSl1DvAKRpgXNn+Z7ZtSinsv680/b80m0m5h0foDXPCH5fxi7jq2HqwKdXlCiA7ipIGutfYC9wJLgC3A+1rrTUqpZ5RSVwdWex6IBOYqpdYqpRYeZ3NntdH9k/nwJ6MYNzAFj9/PvNX5jHlxBbe+9h1fbi+WphghxBlRoQqR7OxsnZOTE5J9twV7Sg7z2pe7eT8nn1qP8Vi7vilRTL+wB1cNTsNmkXu+hBDHUkqt1lpnN7pMAj20ymvcvP3tXmZ9nRfsByY52s7t52fww+FdiQmzhrhCIURbIoHeDri8Pj5au59Xv9jFtkPGNevhNjNTh3Vh2qgMusSHh7hCIURbIIHejmitWbG9mH+u2MWXO4oBMCkYl5XK9At7MKRLbIgrFEKEkgR6O7V5fyWvfrGLhev24w30CTOsexx3XdiDy/p2wmqWdnYhzjYS6O3cwQons77O4+1v91DlNB5OHRduZezAFCYMSmNEjwTMJhXiKoUQrUECvYOodnl5f9U+3v1uL9vr9Q2TGGlnfJYR7tnd4jBJuAvRYUmgd0BbD1axaP1+Fq0/wO7iw8H5KdEOrhyUyoRBqQzpEotSEu5CdCQS6B2Y1ppN+yv59/r9LFp3gILy2uCyznFhXDkolasGpTEgLVrCXYgOQAL9LKG1Zu2+cv697gD/2bCfQ5VHutTJSIxgwqBUrhqcRmZyVAirFEKcCQn0s5Dfr8nZU8ai9ftZvOEAxdXu4LLM5EjGDkjhigEpcuQuRDsjgX6W8/r8fLu7lEXr9/PxxoOU1xx5glJ6bBhXDEhmzIAUsrvFYZFLIYVo0yTQRZDH52flzhKWbDrIp5sPUVh1pFkmPsLG5X07MWZAChf0TsRhNYewUiFEYyTQRaP8fs3a/HKWbDrIfzcdOupqmXCbmUv6JHFF/xQu7dtJ+pQRoo2QQBcnpbVme2E1/910kCWbDrGhoCK4zGJSjOyZwBUDUriifzLJ0Y4QVirE2U0CXZyygvLaQLgf5Lvdpfjr/TMZ0iWWC3olMqJHAkO7xRFmk6YZIVqLBLo4I6WH3Xy25RBLNh3ii+1FuLz+4DKrWTG4cywjeiQwokcC53aLJdxmCWG1QnRsEuii2dS4vazcWcI3u0r4Zlcpm/ZXHHX0bjEpBneJZUSP+OARvAS8EM1HAl20mIpaDzl5pRLwQrQSCXTRaiqddQFvhPzGgmMDfmB6DN0SwkmNCSMt1kFKtIO02DBSYxzER9jkRichTkACXYTMyQK+IbvFRGqMg9SYMFJjHcHxtNjAMCaM6DCLhL44a50o0OVvX9Gioh1WLuubzGV9kwEj4DcWVLC/3MmB8lr2Vzg5UFHLwQon+8trqXR6ySupIa+k5rjbjLCZ6ZUcRb+UKPoEXn1ToomPsLXWxxKiTZJAF60q2mHl/J6Jx11+2OXlQEWtEfgVtRyocHKg3Mn+4Hgth90+1u0rZ92+8qPe2ynKTp+UKPqlRtMn2Qj6Xp0i5Y5XcdaQQBdtSoTdQq9OUfTq1HiPkFprymo8bD1YxdaDleQerCL3YBXbDlVRWOWisMrFF9uLg+ubTYqMxAgj6FOi6JMSTWZyJPERNiLt0nQjOhZpQxcdgt+v2VdWQ+7BqkDYV7HlYCV5xYeP22ZvNimiHRZiwqxEh1mPHjqMoTHPEhyPCbMSG26TrhBEyEgbuujwTCZFt4QIuiVEMGZASnC+0+NjR2G1cSR/oJKth6rYWVhNea2HGrePshoPZfV6n2yqrvHhZHePY1j3eLK7xdEzKVIe/SdCTo7QxVnL7fVT6fRQUeuhstYY1o1XOr3GdE1gnvPI8uJqF06P/6htxYZbGdo1juzu8QzrHsfA9BhpuxctQo7QhWiEzWIiMdJOYqT9lN7n9fnJPVjFqrxScvaUkZNXyqFKF5/lFvJZbqGxbbOJQZ1jyA4cwQ/tFkecXIUjWpgcoQtxhrTW5JfVkrOnlFV5ZazOK2Proapj1uvVKZJh3eMY3DkWk1LUuL3UevzUur3UuH3UenzUBoYNp2vdvsD6PlxeP1aTCbvFhN1qwmY2YbeajWmLCbvFjM1yZLndYg6sY8wLs5rJSIqgX2o0PZMiscpDTdoVubFIiFZWUeNhzd4y4yg+r4y1+eW4vf6Tv7GVWc2KXp2MK4D6pUbTN9UYnupfLU1x2OXlUKWT0sNuUmPDSItxyFVGp0ECXYgQc3l9bCyoZPWeUjbvr8RiNhFuMxNmNRNmM9cbtxBmNaYdgWH98TCbGbvFjNfvx+X14/L4cXl9uL2Baa8fV+Ao/sg8X3C+2+enyull+6FqthysZM9xbuBKjLTTLxDu/VKNG7d6JkVisxx7NO/2+imscnKo0kVhpZODlQ3HnRRWuqhyeY96X7jNTM+kSHp3iqRnJ2PYq1MkXePDW+RRiNUuL4WBXygJkXbSY8Ma/Txt3RkHulJqLPAXwAy8qrX+fYPldmA2MBQoAaZqrfNOtE0JdCFC77DLG7iWv5ItByrZcsC4Guiw23fMulazomeSEbqHXV4OBkK75LC7kS0fy24xkRLjIDbcRkFZLcXVrkbXs5lNZCRG0KtB0GckRhxzollrTZXLS2GgFuNehMAvlCoXhyqdFFUZyxp+JpOC1JgwuiWE0y0hnC7x4XSLjwiOn+mlqV6fn5LDboqqXEde1cawf1o012d3Oa3tnlGgK6XMwDZgNJAPrAJu1FpvrrfOj4FBWusfKaVuAK7VWk890XYl0IVom/x+45zAlkDI5x6oOuHRvNmkSIq0kxxtJznaEXgdGU+JcZAc5TimD57yGjc7CqvZUVjN9sBwR2E1BeW1je7HpIzLRbslRFDj9gYDu+EVR8fjsJroFOUgLsJGcZWLAxW1J+xXKDbcStf48MA+wwPjEXSOC6PG7aO4+tigrnsVV7sorXFzvHgdn5XC328a2qS6GzrTQB8JzNBajwlMPwqgtf5dvXWWBNZZqZSyAAeBJH2CjUugC9G+1B3N5xUfJjrMSkoguBMi7Zib8Rr8wy4vu4oOs72wKhj2OwuryStp/CaxMKuZ5Gg7naIddIqy0ynKEZi2kxzloFO0naQoB9GOo3+huL1+Cspr2VNymL2lNewtqWFPYLi3tIZaz7F/pZwKk4L4CDtJUYFX5JHxvilRjOp1/C4wTuRML1tMB/bVm84Hhh9vHa21VylVASQAxfVXUkrdDdwN0LVr1yYVL4RoGyLsFoYGLsFs6f1kdY4hq3PMUfNdXh95xTXsKTlMlMNKp2g7naLsp92Fg81iNO1kJEYcs0xrTVG1Kxjue0pqMm5lnQAABe9JREFU2FdqBP7+8loi7JZgQCfWC+r6wR0fYWvWX3RN0arXoWutZwIzwThCb819CyHaN7vFHOxds6UppegU5aBTlIPs7vEtvr/m0pRTvAVA/db7zoF5ja4TaHKJwTg5KoQQopU0JdBXAb2VUhlKKRtwA7CwwToLgdsC45OBZSdqPxdCCNH8TtrkEmgTvxdYgnHZ4mta601KqWeAHK31QuBfwJtKqR1AKUboCyGEaEVNakPXWi8GFjeY91S9cScwpXlLE0IIcSra321SQgghGiWBLoQQHYQEuhBCdBAS6EII0UGErLdFpVQRsOc0355Ig7tQ2xip78xIfWeurdco9Z2+blrrpMYWhCzQz4RSKud4fRm0BVLfmfn/7Z1NaB1VFMd/f7Aq1GITCrWCIHUh6EINpVSppVCJbZBGXUhFMFo3RQt2IRIolOKuii4UUfwoVika/KiKtNgogqtEsKRftpq0ZGFJk4XSKi60elzc+2CczLw88zofeZwfDHPf3DOZP2fOPW/mzLxc19c+ddfo+orBSy6O4zgdgid0x3GcDmGhJvQ3qhYwB66vPVxf+9Rdo+srgAVZQ3ccx3Fms1Cv0B3HcZwUntAdx3E6hFondEkbJf0oaULSYEb/VZKGYv+opBtL1HaDpG8k/SDppKSnM2zWS7ogaSwuu7L+VoEaJyUdj8eeNd+fAi9H/x2T1FOitpsTfhmTdFHSjpRN6f6TtFfSjKQTiW3dkoYljcd15pQ9kgaizbikgSybArS9IOl0PH8HJC3N2bdpLBSscbekc4nz2Jezb9PxXqC+oYS2SUljOfuW4sO2MLNaLoR/1XsGWAlcCRwFbknZPAm8HttbgKES9a0AemJ7CWEi7bS+9cAXFfpwEljWpL8POAQIWAOMVniuzxN+MFGp/4B1QA9wIrHteWAwtgeBPRn7dQNn47ortrtK0NYLXBHbe7K0tRILBWvcDTzTQgw0He9F6Uv1vwjsqtKH7Sx1vkJfDUyY2Vkz+xP4AOhP2fQD+2L7I2CD5jO54DwwsykzOxLbvwGnCHOrLiT6gXctMAIslbSiAh0bgDNmNt9fDl82zOxbwv/0T5KMs33A/Rm73gsMm9kvZvYrMAxsLFqbmR02s0vx4whhRrHKyPFfK7Qy3tummb6YOx4C3r/cxy2LOif0rMmp0wnzP5NTA43JqUsllnruAEYzuu+UdFTSIUm3lioMDDgs6fs4QXeaVnxcBlvIH0RV+q/BcjObiu3zwPIMmzr4civhjiuLuWKhaLbHstDenJJVHfx3NzBtZuM5/VX7cE7qnNAXBJKuAT4GdpjZxVT3EUIZ4TbgFeDTkuWtNbMeYBPwlKR1JR9/TuK0hpuBDzO6q/bfLCzce9fuXV9JO4FLwP4ckypj4TXgJuB2YIpQ1qgjD9P86rz246nOCb32k1NLWkRI5vvN7JN0v5ldNLPfY/sgsEjSsrL0mdm5uJ4BDhBua5O04uOi2QQcMbPpdEfV/ksw3ShFxfVMhk1lvpT0GHAf8Ej8wplFC7FQGGY2bWZ/m9k/wJs5x640FmP+eBAYyrOp0oetUueEXuvJqWO97W3glJm9lGNzXaOmL2k1wd+lfOFIWixpSaNNeHh2ImX2OfBofNtlDXAhUVooi9yroir9lyIZZwPAZxk2XwK9krpiSaE3bisUSRuBZ4HNZvZHjk0rsVCkxuRzmQdyjt3KeC+Se4DTZvZzVmfVPmyZqp/KNlsIb2H8RHj6vTNue44QvABXE27VJ4DvgJUlaltLuPU+BozFpQ/YBmyLNtuBk4Qn9iPAXSXqWxmPezRqaPgvqU/Aq9G/x4FVJZ/fxYQEfW1iW6X+I3y5TAF/Eeq4TxCey3wNjANfAd3RdhXwVmLfrTEWJ4DHS9I2Qag9N2Kw8dbX9cDBZrFQov/ei/F1jJCkV6Q1xs+zxnsZ+uL2dxpxl7CtxIftLP7Tf8dxnA6hziUXx3Ec53/gCd1xHKdD8ITuOI7TIXhCdxzH6RA8oTuO43QIntAdx3E6BE/ojuM4HcK/5fY8HvRicZQAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Three hidden layers\n", "# Not really worth it, very similar results\n", "# Overfits even faster\n", "model = models.Sequential()\n", "model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(1, activation='sigmoid'))\n", "model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n", "history = model.fit(partial_x_train, partial_y_train,\n", " epochs=20, batch_size=512, verbose=0,\n", " validation_data=(x_val, y_val))\n", "pd.DataFrame(history.history).plot(lw=2);" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Set the learning rate to 0.1 and plot the learning curves again.\n", "# learning rate 0.4 gives very high losses which don't plot nicely\n", "# For high learning rates there is no convergence, the loss actually increases\n", "from tensorflow.keras import optimizers\n", "\n", "model = models.Sequential()\n", "model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "model.compile(optimizer=optimizers.RMSprop(lr=0.1),\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])\n", "history = model.fit(partial_x_train, partial_y_train,\n", " epochs=10, batch_size=512, verbose=0,\n", " validation_data=(x_val, y_val))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXyU1b348c93lmSy72xhCbiBJgQkKIIial0v7nXXVlrrT1uldWu9LtWr9Gqrrde21kq9qFjXit5a61ItWhTQEhAMIiJigIQl+0aW2c7vj2cymYSETGDCJJPvu53X88x5lvlm1O9z5jznOUeMMSillIpdtmgHoJRSqn9poldKqRiniV4ppWKcJnqllIpxmuiVUirGOaIdQHeys7NNXl5etMNQSqlBY/Xq1VXGmJzutg3IRJ+Xl0dxcXG0w1BKqUFDRLb2tE2bbpRSKsZpoldKqRiniV4ppWLcgGyjV0oNHB6Ph7KyMlpbW6MdigJcLhejR4/G6XSGfYwmeqXUPpWVlZGSkkJeXh4iEu1whjRjDNXV1ZSVlTF+/Piwj9OmG6XUPrW2tpKVlaVJfgAQEbKysvr860oTvVKqV5rkB479+WcRM4m+pKye7yz6N3e+VhLtUJRSakCJmTZ6p0NYtqmS3PSEaIeilIqw5ORkmpqaoh3GoBUzNfrDh6WQ4nJQXtfCzvqWaIejlFIDRswkeptNmDYuA4Di0tooR6OU6g/GGG677Tby8/MpKCjgpZdeAmDnzp3Mnj2bKVOmkJ+fz4cffojP5+Pqq68O7vvII49EOfroiZmmG4CicRl88GUlxaU1nF04KtrhKBVz8m7/e7+ct/TB/whrv1dffZW1a9eybt06qqqqmD59OrNnz+b555/n9NNP584778Tn89Hc3MzatWspLy9n/fr1ANTV1fVL7INBzNToAYryMgEo3qo1eqVi0UcffcRll12G3W5n+PDhnHjiiaxatYrp06fz1FNPce+991JSUkJKSgoTJkxgy5Yt3Hjjjbz99tukpqZGO/yoiakafeHodBw24YudDTS1eUmOj6k/T6moC7fmfbDNnj2bZcuW8fe//52rr76am2++me985zusW7eOd955hz/+8Y+8/PLLLFq0KNqhRkVM1egT4uzk56bhN/DpNq3VKxVrTjjhBF566SV8Ph+VlZUsW7aMY445hq1btzJ8+HB+8IMfcM0117BmzRqqqqrw+/1ceOGFLFiwgDVr1kQ7/KiJuSpv0bgM1m6vY1VpLScc1u0Y/EqpQer8889n5cqVFBYWIiL86le/YsSIETzzzDM89NBDOJ1OkpOTWbx4MeXl5cybNw+/3w/AAw88EOXoo0eMMfveQWQMsBgYDhhgoTHm0S77CPAocBbQDFxtjFkT2PZd4K7ArguMMc/0FlRRUZHZ34lH3l6/i+v+vJpZh2bx3DUz9uscSqkOX3zxBZMmTYp2GCpEd/9MRGS1Maaou/3DqdF7gVuMMWtEJAVYLSLvGmM2hOxzJnBY4HUs8DhwrIhkAvcARVgXidUi8roxpt/aVdq7WH66rQ6vz4/DHlOtU0op1We9ZkFjzM722rkxphH4Asjtstu5wGJj+RhIF5GRwOnAu8aYmkByfxc4I6J/QRc5KfGMz06i2e3ji52N/flRSik1KPSpuisiecBU4JMum3KB7SHvywJlPZV3d+5rRaRYRIorKyv7EtZe2mv1q0prDug8SikVC8JO9CKSDCwBfmKMaYh0IMaYhcaYImNMUU7Ogd1EnZ5nJfrV2p9eKaXCS/Qi4sRK8s8ZY17tZpdyYEzI+9GBsp7K+9W0cdaDU6tKa+jtZrNSSsW6XhN9oEfN/wJfGGN+08NurwPfEcsMoN4YsxN4BzhNRDJEJAM4LVDWrw7JSSIj0UlFYxtltTrAmVJqaAun180s4CqgRETWBsruAMYCGGP+CLyJ1bVyM1b3ynmBbTUicj+wKnDcfcaYfm84FxGmjcvkvS92s6q0hjGZif39kUopNWD1muiNMR8B+5zSxFjtIz/qYdsi4KA/dzw9L4P3vthN8dZaLjh69MH+eKXUIOT1enE4Yu450tgaAiFUUV77kMXa80apWHDeeecxbdo0jjrqKBYuXAjA22+/zdFHH01hYSGnnHIKAE1NTcybN4+CggImT57MkiVLAGvyknavvPIKV199NQBXX3011113Hcceeyw//elP+fe//81xxx3H1KlTmTlzJl9++SUAPp+PW2+9lfz8fCZPnszvfvc7li5dynnnnRc877vvvsv5559/ML6OPom9S1dAfm4acQ4bm3Y3Ud/sIS3RGe2QlBr87k3rp/PW97rLokWLyMzMpKWlhenTp3Puuefygx/8gGXLljF+/HhqaqxK3f33309aWholJda0orW1vfe+KysrY8WKFdjtdhoaGvjwww9xOBy899573HHHHSxZsoSFCxdSWlrK2rVrcTgc1NTUkJGRwQ9/+EMqKyvJycnhqaee4nvf+96BfRf9IGYTfbzDTuHoNFaV1rJ6Ww0nTxwe7ZCUUgfgt7/9La+99hoA27dvZ+HChcyePZvx48cDkJlp9bZ77733ePHFF4PHZWRk9Hruiy66CLvdDkB9fT3f/e53+eqrrxARPB5P8LzXXXddsGmn/fOuuuoq/vznPzNv3jxWrlzJ4sWLI/QXR07MJnqwxqdfVVpLcWmtJnqlIiGMmnd/+OCDD3jvvfdYuXIliYmJzJkzhylTprBx48awz2F1ILS0trZ22paUlBRcv/vuuznppJN47bXXKC0tZc6cOfs877x58zj77LNxuVxcdNFFA7KNP2bb6MEayRJ0akGlBrv6+noyMjJITExk48aNfPzxx7S2trJs2TK++eYbgGDTzamnnspjjz0WPLa96Wb48OF88cUX+P3+4C+Dnj4rN9d6gP/pp58Olp966qk88cQTeL3eTp83atQoRo0axYIFC5g3b17k/ugIiulE3z4UwrqyOtq8vihHo5TaX2eccQZer5dJkyZx++23M2PGDHJycli4cCEXXHABhYWFXHLJJQDcdddd1NbWkp+fT2FhIe+//z4ADz74IHPnzmXmzJmMHDmyx8/66U9/yn/+538yderUYFIHuOaaaxg7diyTJ0+msLCQ559/PrjtiiuuYMyYMQN2lM9ehymOhgMZprirU3/zL76qaGLJ9TODiV8pFT4dprh3N9xwA1OnTuX73//+Qfm8vg5THNM1euiYR3b1Vu1mqZSKvGnTpvHZZ59x5ZVXRjuUHsV+og+OZKnt9EqpyFu9ejXLli0jPj4+2qH0KPYTfchIlgOxmUoppfpbzCf6sZmJ5KTEU7PHzZaqPdEORymlDrqYT/QiEmy+Wa3NN0qpISjmEz103JDVGaeUUkPR0Ej043TGKaXU0DUkEv2Ro1JJcNrZUrWHqqa2aIejlOpnoSNVdlVaWkp+fv5BjCb6hkSid9ptTBmTDmitXik19Ay80Xf6yfS8DFZuqaa4tIbTjxoR7XCUGpQKninol/OWfLdkn9tvv/12xowZw49+ZM1vdO+99+JwOHj//fepra3F4/GwYMECzj333D59bmtrK9dffz3FxcU4HA5+85vfcNJJJ/H5558zb9483G43fr+fJUuWMGrUKC6++GLKysrw+XzcfffdwWEXBrpeE72ILALmAhXGmL1+74jIbcAVIeebBOQEphEsBRoBH+Dt6fHcg2Fa4IZssdbolRp0LrnkEn7yk58EE/3LL7/MO++8w/z580lNTaWqqooZM2ZwzjnndBqlsjePPfYYIkJJSQkbN27ktNNOY9OmTfzxj3/kxz/+MVdccQVutxufz8ebb77JqFGj+Pvf/w5Yg58NFuHU6J8Gfg90O8iyMeYh4CEAETkbuKnLvLAnGWOqDjDOA3b02HRsAuvL62n1+HA57dEOSalBp7ead3+ZOnUqFRUV7Nixg8rKSjIyMhgxYgQ33XQTy5Ytw2azUV5ezu7duxkxIvxf7B999BE33ngjABMnTmTcuHFs2rSJ4447jl/84heUlZVxwQUXcNhhh1FQUMAtt9zCz372M+bOncsJJ5zQX39uxPXaRm+MWQaE2y/xMuCFA4qon6S4nBwxIhWPz7Bue120w1FK9dFFF13EK6+8wksvvcQll1zCc889R2VlJatXr2bt2rUMHz58r3Hm99fll1/O66+/TkJCAmeddRZLly7l8MMPZ82aNRQUFHDXXXdx3333ReSzDoaI3YwVkUTgDGBJSLEB/iEiq0Xk2l6Ov1ZEikWkuLKyMlJhdTK9fR5Zbb5RatC55JJLePHFF3nllVe46KKLqK+vZ9iwYTidTt5//322bt3a53OecMIJPPfccwBs2rSJbdu2ccQRR7BlyxYmTJjA/PnzOffcc/nss8/YsWMHiYmJXHnlldx2222sWbMm0n9iv4nkzdizgeVdmm2ON8aUi8gw4F0R2Rj4hbAXY8xCYCFYwxRHMK6gaeMyWLxyq04YrtQgdNRRR9HY2Ehubi4jR47kiiuu4Oyzz6agoICioiImTpzY53P+8Ic/5Prrr6egoACHw8HTTz9NfHw8L7/8Ms8++yxOp5MRI0Zwxx13sGrVKm677TZsNhtOp5PHH3+8H/7K/hHWePQikge80d3N2JB9XgP+Yox5voft9wJNxpiHe/u8SI5HH2pHXQszH1xKqsvB2p+fhs0W/k0bpYYqHY9+4InKePQikgacCPw1pCxJRFLa14HTgPWR+Lz9NSo9gVFpLhpavXxV0RTNUJRS6qAJp3vlC8AcIFtEyoB7ACeAMeaPgd3OB/5hjAkdHnI48Fqgq5MDeN4Y83bkQt8/RXmZvL5uB6tKazhiREq0w1FK9ZOSkhKuuuqqTmXx8fF88sknUYooenpN9MaYy8LY52msbpihZVuAwv0NrL8U5WXw+rodrN5ay5UzxkU7HKVUPykoKGDt2rXRDmNAGBJDIIQqGqcjWSqlhpYhl+iPGJFCSryDstoWdjdEps+tUkoNZEMu0dttwtTAsMXFOhGJUmoIGHKJHkInDNfmG6VU7BuaiT5PJyJRKpbtazz6oWhIJvopY9Kx24QNOxvY0+aNdjhKqRjl9Q6M/DJkxqMPlRjnIH9UKuvK6lm7vY5Zh2ZHOySlBoUvJvbPE7KTNn6xz+2RHI++qamJc889t9vjFi9ezMMPP4yIMHnyZJ599ll2797Nddddx5YtWwB4/PHHGTVqFHPnzmX9eusZ0Icffpimpibuvfde5syZw5QpU/joo4+47LLLOPzww1mwYAFut5usrCyee+45hg8fTlNTEzfeeCPFxcWICPfccw/19fV89tln/M///A8Af/rTn9iwYQOPPPLIfn+3MEQTPcC0cZmsK6tnVWmNJnqlBrhIjkfvcrl47bXX9jpuw4YNLFiwgBUrVpCdnU1NjXUPb/78+Zx44om89tpr+Hw+mpqaqK3dd7Ov2+2mfRiX2tpaPv74Y0SEJ598kl/96lf8+te/5v777yctLY2SkpLgfk6nk1/84hc89NBDOJ1OnnrqKZ544okD/fqGbqKfnpfBouXfaDu9Un3QW827v0RyPHpjDHfcccdexy1dupSLLrqI7Gyr4peZaT1zs3TpUhYvtqbjsNvtpKWl9ZroQ2eeKisr45JLLmHnzp243W7Gjx8PwHvvvceLL74Y3C8jw7p3ePLJJ/PGG28wadIkPB4PBQUHPqvXkE300wI3ZNdsrcXr8+OwD8nbFUoNGu3j0e/atWuv8eidTid5eXlhjUe/v8eFcjgc+P3+4PuuxyclJQXXb7zxRm6++WbOOeccPvjgA+699959nvuaa67hv//7v5k4cSLz5s3rU1w9GbLZbViKi7GZiexx+9i4qzHa4SilehGp8eh7Ou7kk0/mL3/5C9XV1QDBpptTTjklOCSxz+ejvr6e4cOHU1FRQXV1NW1tbbzxxhv7/Lzc3FwAnnnmmWD5qaeeymOPPRZ83/4r4dhjj2X79u08//zzXHZZryPQhGXIJnro6Gap49MrNfB1Nx59cXExBQUFLF68OOzx6Hs67qijjuLOO+/kxBNPpLCwkJtvvhmARx99lPfff5+CggKmTZvGhg0bcDqd/PznP+eYY47h1FNP3edn33vvvVx00UVMmzYt2CwEcNddd1FbW0t+fj6FhYW8//77wW0XX3wxs2bNCjbnHKiwxqM/2PprPPqunv9kG3e8VsLcySP5/eVH9/vnKTUY6Xj0B9/cuXO56aabOOWUU7rdHpXx6Aer4NSCpbUMxAueUmpoqaur4/DDDychIaHHJL8/huzNWIBDcpJJS3Cyq6GV8roWRmckRjskpVSEDMbx6NPT09m0aVPEzzukE73NJhSNy+CfGysoLq3VRK9UD4wxvfZPH2hidTz6/Wl9GNJNN9DRzbJ4q96QVao7LpeL6upqbd4cAIwxVFdX43K5+nRcOFMJLgLmAhXdTQ4uInOw5or9JlD0qjHmvsC2M4BHATvwpDHmwT5FdxBMz7MeitAhi5Xq3ujRoykrK6OysjLaoSisC+/o0aP7dEw4TTdPA78HFu9jnw+NMXNDC0TEDjwGnAqUAatE5HVjzIY+RdjPCnLTiLPb+HJ3I/UtHtISnNEOSakBxel0Bp/mVINTr003xphlwP60axwDbDbGbDHGuIEXgd5HHDrIXE47BaPTMAbWbNNavVIq9kSqjf44EVknIm+JyFGBslxge8g+ZYGybonItSJSLCLFB/snYvtEJKu1+UYpFYMikejXAOOMMYXA74D/25+TGGMWGmOKjDFFOTk5EQgrfEV5OmG4Uip2HXCiN8Y0GGOaAutvAk4RyQbKgTEhu44OlA040wI1+nVldbi9/l72VkqpweWAE72IjJBAB1sROSZwzmpgFXCYiIwXkTjgUuD1A/28/pCZFMchOUm0evx8vqM+2uEopVREhdO98gVgDpAtImXAPYATwBjzR+DbwPUi4gVagEuN1eHWKyI3AO9gda9cZIz5vF/+iggoGpfJ15V7WL21lqljIzOQkFJKDQS9JnpjzD7HyTTG/B6r+2V3294E3ty/0A6uorwMXirezqrSGq45YUK0w1FKqYgZ8k/Gtmu/Ibt6qw5wppSKLZroA/KyEslOjqOqyU1pdXO0w1FKqYjRRB8gIsHeNzoRiVIqlmiiD6Hj3iilYpEm+hDBGr2OZKmUiiGa6EMcNSoNl9PG15V7qNnjjnY4SikVEZroQ8Q5bBSOTges3jdKKRULNNF30dFOr803SqnYoIm+i44Zp7RGr5SKDZrouzh6bAYiUFJWT6vHF+1wlFLqgGmi7yItwckRw1Nw+/yUlOsAZ0qpwU8TfTfau1nq+PRKqVigib4b7TdkdcYppVQs0ETfjY4Hp2rx+3WAM6XU4KaJvhujMxIYkeqivsXD15VN0Q5HKaUOiCb6bohIsJvlKm2+UUoNcproezBdx71RSsWIXhO9iCwSkQoRWd/D9itE5DMRKRGRFSJSGLKtNFC+VkSKIxl4fyvSkSyVUjEinBr908AZ+9j+DXCiMaYAuB9Y2GX7ScaYKcaYov0LMTomjkghKc7OtppmKhpaox2OUkrtt14TvTFmGdBj+4UxZoUxpr3a+zEwOkKxRZXDbgtOEq7DISilBrNIt9F/H3gr5L0B/iEiq0Xk2n0dKCLXikixiBRXVlZGOKz9U9Q+7o023yilBjFHpE4kIidhJfrjQ4qPN8aUi8gw4F0R2Rj4hbAXY8xCAs0+RUVFA6LzetG4QDu93pBVSg1iEanRi8hk4EngXGNMdXu5MaY8sKwAXgOOicTnHSxTxqZjtwmf72ig2e2NdjhKKbVfDjjRi8hY4FXgKmPMppDyJBFJaV8HTgO67bkzUCXHO5g0MgWf37B2W120w1FKqf0STvfKF4CVwBEiUiYi3xeR60TkusAuPweygD906UY5HPhIRNYB/wb+box5ux/+hn7V0Xyj7fRKqcGp1zZ6Y8xlvWy/Brimm/ItQOHeRwwuRXkZPL2iVEeyVEoNWvpkbC/aa/SfbqvDpwOcKaUGIU30vRiR5mJ0RgJNbV427mqIdjhKKdVnmujDEByfXtvplVKDkCb6MATHp9cHp5RSg5Am+jBMDw5wpjdklVKDjyb6MBw2LJlUl4Md9a2U17VEOxyllOoTTfRhsNkkpPlGa/VKqcFFE32YivSGrFJqkNJEH6aicTq1oFJqcNJEH6bCMek47cKXuxpoaPVEOxyllAqbJvowuZx28nPT8BvrKVmllBosNNH3QXvzzWq9IauUGkQ00ffBtMC4N9pOr5QaTDTR90H71IJrt9fh8fmjHI1SSoVHE30fZCfHMz47iRaPjw07dIAzpdTgoIm+j9rb6XUiEqXUYKGJvo/am2/0CVml1GARVqIXkUUiUiEi3c75KpbfishmEflMRI4O2fZdEfkq8PpupAKPlvYnZIu31mKMTkSilBr4wq3RPw2csY/tZwKHBV7XAo8DiEgmcA9wLHAMcI+IZOxvsAPBhOwkMpPiqGxsY1tNc7TDUUqpXoWV6I0xy4B9tVWcCyw2lo+BdBEZCZwOvGuMqTHG1ALvsu8LxoAnIjo+vVJqUIlUG30usD3kfVmgrKfyvYjItSJSLCLFlZWVEQqrf3TckNV2eqXUwDdgbsYaYxYaY4qMMUU5OTnRDmefgu30WqNXSg0CkUr05cCYkPejA2U9lQ9q+bmpxDlsfFXRRF2zO9rhKKXUPkUq0b8OfCfQ+2YGUG+M2Qm8A5wmIhmBm7CnBcoGtXiHnSmj0wEdn14pNfCF273yBWAlcISIlInI90XkOhG5LrDLm8AWYDPwJ+CHAMaYGuB+YFXgdV+gbNCblqfj0yulBgdHODsZYy7rZbsBftTDtkXAor6HNrBNz8vgcWC13pBVSvXGGPC2gacZvK3gabHWPa2BZQt4W6zlpLMhPiWiHx9Wold7O3qsVaNfV1ZPm9dHvMMe5YiUUn1mjJVc3U0dCTeYgEOSb09JuduykFdoGWE+YDn6GE30A0V6YhyHD09m0+4m1pfXB4cwVkodJH6/laDbGqC1ofOy27LGwHp95zK/9+DEa48DZwI4Eqxl8JUIDlfHutMV8Y/WRH8Apo3LZNPuJlaV1mqiVypcPm9HE4Z7j5Vs90rI9T0k6S5l4daS98UeD/HJ4EwKJFtXIOGGJuXuyhI7yrsm6r32TQBb9H71a6I/ANPzMnjh39us/vQnRjsapQ6A39d9c0O370OaK/q0T6AskjXouGSrmSM+FVypIcv2srRuygL7ta874iMXzwClif4AFAVq8au31mCMQUSiHJFSXbibobYUar+xljXfBNa3WjXi9gTtO4jPg4gtUHsO1ILbk258Spdk3V6W1k1ZYD2KteTBRBP9ARiTmcCwlHgqGtv4unIPhw5LjnZIaqgxBvZUWcm75puOpN6e0Jt2h3ki6Whi2KsNOcyycPexO0ErRQeVJvoDICIU5WXwZskuiktrNNGr/uHzQP32kNp4aUhSL7VuSHZh/GD8giEOkzIGkzIak5SLSR6FSRyBSRhm1Ywd8Vb7st1p1bTbtSfi9kXwfQ9LQvY3IB4BT/smH9AEsmfvYwTE6UQcjr2WOJ36KzlCNNEfoKJxmVai31rLpceMjXY4agAzHg97Vq7Es2MnxuPpeLndmJYmTFM1prEas6cO01yPaWm0yttaMT5jJW6/dCRxPxhfIsYkY4wd47dZZV6/VdMPagG+CrwGmR4uAuJ0Is7AxcDpRBzOHvZxgtPR/T7OkH3sdhCxLiwiWFcg60LUfXnHhS24rctxPZV3Ol9wW+DvFSFp+nRsSUkR/Ro10R8gnXFK9abt66+pW/Iq9X/9K77q6v04Qzg3Cw3g63grgsTFWYmsfdn+iouzasw2W8cFIfTCEFg37T1aTOfyHpeBHTtNyNPLscb4weuzLnheb6clXi+0XwxbWsL4DmLDhDf/TvyECRE9pyb6A3TkyFQS4+yUVjdT2dhGTkrs38FXvfM17aHx7beoe2UJLWvXBsvjUj0kZrsRO4jNBF4gDjuSlIGkZCEp2UjacOuVPgpJG4EkJu+drHt6HxeH2Af/TUpjTEei73IRsF7tF4Ru9nGHlrkx7ReN4PEh5/F6MV6PdfExBJYGMFYMXcsD24IxdrOtT+WhF1ZjsCUmRvy71ER/gBx2G1PGpLPi62pWb63hjPyR0Q5JRYkxhpY1a6h7ZQkNb78drIXa4mykjm4kfUIzrhFO5IgzIOsQyMiDjPGQOR6SR1g1bBUkIhAXh8TFRTuUQU8TfQQU5WWy4utqiktrNdEPQZ6KCur/+lfql7yKu7Q0WJ6Ql0r68G2kjmnBFh8H038Ax98EycOiF6wakjTRR0D7jFOrdMjiIcN4PDT961/ULXmVpmXLwGe1jzuyM0krSCUtsZj4lB1gc8DR34MTboG0bidXU6rfaaKPgKlj07EJfF5eT4vbR0Lc4G8fVd3r9saqw0HKnBNIO6SV5Oa3EOO2uioWXgkn3mY10SgVRZroIyDF5WTiiFQ27Gxg7fY6jjskK9ohqQjq8cbqIYeQfvYZpGWX4tj4LOxpBQTyvw1zbofsw6IXtFIhNNFHSFFeBht2NrB6a40m+hjQ443VpCRSzzqL9Lmn4Wp6H/nkAdgVeGBp0tkw5w4YfmQUI1dqb5roI6QoL5PFK7fqjFODXI83VoumkX7ht0mdMxPbZ8/A0sut4W4BDjsNTroDRk2NTtBK9SKsRC8iZwCPAnbgSWPMg122PwKcFHibCAwzxqQHtvmAksC2bcaYcyIR+EDTfkN2zbZafH6D3aaPbg8WwRurryyh6cMPO26s5uSQdt55pF1wPvG5w2HVk7DwJmgJPBw3fjacdBeMPTaK0SvVu14TvYjYgceAU4EyYJWIvG6M2dC+jzHmppD9bwRCqzYtxpgpkQt5YBqVnkBuegLldS1s2t3IpJGp0Q5J9aLHG6unfou0Cy8k+fjjEXyw+mlY8uuOAcLGHAsn32UleqUGgXBq9McAm40xWwBE5EXgXGBDD/tfBtwTmfAGl2njMiiva6F4a60m+gHK17SHhrfepH7Jq3vfWL3wQtLOPQdHVpY1kNjaP8O/HoKGMmunkVPg5Lvh0FN09EU1qIST6E1uNhwAABwmSURBVHOB7SHvy4Buf6uKyDhgPLA0pNglIsWAF3jQGPN/PRx7LXAtwNixg3NwsOl5Gby+bgfFpTVcNWNctMNRIVo3bKDm2T93f2P12xfimjzZehLT74O1L8C/HrRGhgQYdhScfCcccZYmeDUoRfpm7KXAK8aYkNGVGGeMKReRCcBSESkxxnzd9UBjzEJgIUBRUVEE5gc7+NqnEyw+iDdkvT4/9S2evV4NLR6a3T5SXE4yk5ykJ8aRkRhHRpKT9IQ44hxD43F74/NR/ac/Ufm73wfb3hOLiki78EJSTz+tY1wRvx8+fw0+eACqNlllWYfBSf8JR56vwxOoQS2cRF8OjAl5PzpQ1p1LgR+FFhhjygPLLSLyAVb7/V6JPhYcMSKFlHgH5XUt7KxvYWRaQljH7StZd1de19yxbY/b1/sHdCM53kFGkpOMxLjARcBab78YtK+nJzrJTLLWB9uDYJ7du9nx05/R/MknAGRcfjmZ37mKuLy8jp2MgS/fgvd/AbvXW2Xp46x+8AUXg107pqnBL5x/i1cBh4nIeKwEfylwededRGQikAGsDCnLAJqNMW0ikg3MAn4VicC7avY0s6l2E37jx2d8nZZ+48fn9+HH31Hu77K9yzE+f8d7g+n0fl/7Z+Xtwt3QzPx/fEBOUgp+vxOv14HX68DtcdDmsdPqttPcZqe51UZTi43mNhv44zDGCf44IPzao00gNcFJWsir/X2i005Dq4faZg91zW5q9ripa/ZQ1+Khqc1LU5uX7TXhD/8a77CRmdTlwhBysQj95ZCZGEd6kpOUeEdUJo9oXPo+O++4A19dHfasLEb98pckHz+rYwdj4Ot/wtJfwI41VllqLsy+FaZcCQ4dSEvFjl4TvTHGKyI3AO9gda9cZIz5XETuA4qNMa8Hdr0UeNF0GoyaScATIuLHyl4PhvbWiaTNdZu56q2r+uPUfWODuHTY2Gy9uuUMvJKtL6XrvFRiHNglDoe4cNriibPFk+BIwOVwkehMINmZSHJ8IqnxiaTEJZLoTLC2210kOK2ly+EiNS6VSVmTiLd3HjrZ7zc0tnmp3eOmttlK/jWh681u6prd1O7xUNvsDrw8tHn97KxvZWd9a9hfh8MmjEx3ccrE4ZyZP4KivMx+7Xrqb2uj4uFfU/vsswAkHX88ox58AEd2dsdOpR/B0gWwLVAnSRpmjUUz7WprHlOlYox0zssDQ1FRkSkuLu7TMVvqtnD38ruxiQ2b2LDb7NZSOi+D27uU97R/j8fZ9t5uFztNrX5e+7Qcr/EQ7/ThcHhw2L3Y7B5sdg+IGyNufKYNH248/jbafC20+lpp9bbS4m3pmPAhAlx2F9NGTGPmyJnMyp3FhLQJ+1XDNsbQ4vEFfxXUhvxC6PZisce6WHRtWspOjuf0o4ZzZv5IZkzIxGGPXNt325YtlN98C20bN4LDwbCbbiJz3tVIe/v69lWw9H745l/W+4QMmPUTOOYHEBfZGX2UOthEZLUxpqjbbbGS6GOFMQa33x1M+i3eluB6q7eVFl9HWad9fK17le9u3s3mus2dzj88cTgzR81kZu5Mjht5HGnxaf3697R5fWzY0cDb63fx1vpdbKvp+JmTkejk1COHc2bBSGYdkr3fN4iNMdS/+iq7FvwC09KCc+xYcn/9MAkFBVYTTemHsPxR2PyedUB8Ksy8EY69DlzaDVbFBk30Q1hVSxUrd6xk+Y7lrNyxkprWjikPBSE/O99K/KNmMjlnMg5b/918NMbweSDpv7l+J1sq9wS3pbgcfGuS1bwz+/AcXM7wbvz6GhvZdc89NLz5FgCpZ5/NiHt+jj3BBRv+D1b8Fnaus3Z2JsGM6+C4GyAxM+J/n1LRpIleAeA3fr6s+TKY9NdUrMHr9wa3JzuTOXbkscHEPzpldL/FYozhq4om3irZxVvrd7JxV2NwW2KcnZMmDuOs/JHMOSKHpPjuLz4ta9dSfsuteMrLkcRERt7zc9LOOBnWPAsf/wHqA49/JGbDsf8Pir4PSTrgnIpNmuhVt5o9zRTvLmZ5+XJW7FhBaUNpp+3jUsdx3MjjmJU7i+kjppPk7L927G+q9vDW+p28VbKLkvL6YHm8w8acI3I4M38kJ08aRqrLifH7qf7Tk1T+9rfg8+E68khy77uduF1/h+JF0Bo4PutQq/ZeeCk4w+vqqtRgpYlehaW8qZwVO1awonwFn+z8hEZPRy3bYXMwJWcKs3Jncdyo45iUOQmb9M9DRNtrmgNt+jtZs60uWB5nt3H6CDvf+eBpkj//FIDMi89m2ORGZMNfwO+xdhx7nNUGf/iZ+qCTGjI00as+8/q9rK9az4odK1i+Yznrq9bjN/7g9kxXJjNGzgg28+Qk5vRLHDvrW3gncCPX//Fybln9ImnuPeyJdxF3QhKTs9sHRhVrPPiZ82HM9H6JRamBTBO9OmD1bfV8svOTYOLftWdXp+2HZxweTPpHDz96r777B8LvdlP5619T88xiAGwjbBxy7A4cCX5aTBx/8Z9I8YjLmDZ1GqcfNYIRadoXXg09muhVRBlj+KbhG6s3T/lyincX0+LteMK2ve/+rFGzmDlq5n733Qdo++Ybym+6ibaNX4INhhU0kDmxCZOUzYbRl/JE8xze/saDx9fx7/HRY9M5q2AkZ+SPYHRG4gH/vUoNBproVb9y+9x8WvEpy3csZ0X5Cr6s/bLT9rT4NFKcKSQ4E0h0JJLoSCTBkUCiM7B0JAa3BcvtLtLfXk7SH5Ygbh+S7CN1Vh1ph47GNeNH2KdcHrzB2tDqYekXFbxZspN/baqkzdvRxDR5dBpn5I/gzPyRjM/Wh6JU7NJErw6q9r77K3asYMWOFZ367ocjoc3wg7f9HL/B+nfzoyOFP51uo8XV8avAZXftfbFwJBBnd9HYbGNXnZ8dtX48Xmv8IOOP49C0I3jkvPOYOEIfklKxRxO9ihq/8VPTWkOzp5kWbwvN3mZaPIGlt4VmT3OgrJnmmq9wrfqYY16tI6VBcDsN75yeyCczhtEs0nG8N/yB2Lry1szmh4Xzuf7EwyI6/IJS0bavRK9jsKp+ZRMb2QnZ0FM3dp8XNvwf5pNF1Ly/hYrPUsAI8bmpTHj4IQqn7j1dn9/4g8M87HXBCKy3Dw3RfmGpaK7hzW/ewJG5jD9s3M7bG6/j0W+fwKHDUvr3C1BqANAavYqOtsbAE6yP491Vxo6P09mz2+otk3n5xeTcfie2uMgOFbxm9xrm//Nm6j3V+L0peHdewa2zz+R7x4/XydzVoKdNN2rgaNwFn/wx+ARr0854dvw7C18L2DPSGfnAA6TMmdNvH1/VUsXNH9zGpxXFGGOjreIMJqecw8MXTdGbtWpQ00Svoq/iC1jxe/jsJfB7MD6oKJ1IzaoGABJnzGDUL3+Jc/iwfg/F6/fyu09/x6L1iwDwNBwFlZfws9On8N3j8rBp7V4NQproVf8wBjwtVjOMu8laBteboK3BWi/9CL76R+AgwT3sVMrfaqL1q1Kw28mZP5+sa76P2A/uVIX/3PZP7vzwLvZ4m/C3ZdNSfiXTRx3JQ98uZGyW9r9Xg4smetXB7wfPHisRuwPJOLje2CVp97RPYOluhJBhEfbJkQBTr6S+biK7fv04/uZmnLm51rjxU6b079+8D9satnHTBzexqXYT+ONo2Xk+ca1F3HHWJK44dmxUpkFUan8ccKIXkTOAR7GmEnzSGPNgl+1XAw/RMWn4740xTwa2fRe4K1C+wBjzTG+fd7ATvfH7wevF+P0Yrw/8PozPB76QZad9vBC6b/AYP/i8GL9BnE5s8XGIy4XExWNzxSPx8Ui8yyp3OsMLrj0xu0OSs7sp8L7RWrbXoDtta7IScTBBN3Uk8QjOYIXDBXHJEJ8M8SkQlxKyHlim5uI75Gx2Pfw7Gl7/GwCpZ53JiP/6L+wp0e/10uJtYcHHC3j9a2tWTHfNDNoq5nL8ISP45bcnk5uuI1+qge+AEr2I2IFNwKlAGdZk4ZeFzv0aSPRFxpgbuhybCRQDRVjZZTUwzRhTu6/P3J9E3/rxO+y49dZABVOspbFaF4LrftOx9JvgMipsgjhs2Jw2xC7YHCB2g9gMNrsfER9i82ITr1VuN9gC24PrdoPYCa53lLXvC4hBBBAQMWADcSaAKxmJTwJXEuJKhfhkxJUCCSlIfIqVoIPJeh9J3N77BaulZD3lt9yCZ9s2JCGBEXfdSdoFFwyo2rIxhle+eoUHPnkAj98DbWNp2nY5SfZs7p47iYuLxgyoeJXq6kD70R8DbDbGbAmc7EXgXCCcSb5PB941xtQEjn0XOAN4IZzA+8I01dFW5e19x55I58RoJceQJGkzHclS2Pe+Yl1c/D7B+AXjE/w+ML72dQE/GLcPX5c5VUMComMW8f7SGnhVd/loAbvdmmu1L0u7DeyOvcpbSkrA4yF+4kRyf/0w8Ycc0o9/0/4RES46/CKOzDySmz+4mR1sI/2w39Ow7WJ+tsTLW+t38eAFk3XANDUohZPoc4HtIe/LgGO72e9CEZmNVfu/yRizvYdjc7v7EBG5FrgWYOzYsWGE1Vl80SmMXxiH4AfxIcYPWDVjjA8RPxgvgh8xXmsfv9fahhd8bvB5wB+y7vNYY5x3WndbD/mErvvcgfft615rHJa4ZGvS6fjkwHqgdhyXhLEnYSQev7gwxOM3TgxOjHHg9zswfhvGb8PvBeP24G9rxbS5MW1tHeutrfjdbZjWtm7KrX2DzU6BZijj94HP37m8yxJjrGaqwHcbid88GVdeybDbbsUWH7lRLfvDUdlH8dLcl7j9o9tZXr6cpLFPQe3pfPDlbE575F/ce85RnD81V2v3alCJ1JOxfwNeMMa0icj/A54BTu7LCYwxC4GFYDXd9DUAW3o2rtnn9/WwqAlU/hmID+F3SvyB+xL7ujAYn7/jHkXovYrAe3tmFvETxkf7zwpbuiudP5zyB5747AkeX/s4JuNtRqeXU7bpfG5+eR1vrd/FL87PZ1iK1u7V4BBOoi8HxoS8H03HTVcAjDGhv/2fBH4VcuycLsd+0Ncg1cElNhvYbAjAAK+B9xeb2Li+8HomZ0/mZx/+jPq2Ekbn76b+m8t4dwOsKq3hvnPzOXvySK3dqwEvnArlKuAwERkvInHApcDroTuIyMiQt+cAXwTW3wFOE5EMEckATguUKTUozMqdxctzXyY/K596TwXOsX9g0uGfU9fsZv4Ln/Kj59dQ3dQW7TCV2qdeE70xxgvcgJWgvwBeNsZ8LiL3icg5gd3mi8jnIrIOmA9cHTi2Brgf62KxCriv/casUoPFqORRPHPmM1xyxCV4/B7K7M8yc8Z7JMX7eLNkF6c9soy31++MdphK9UgfmFKqD/729d+4b+V9tPpaGZ96KI7KeazZYj3Re07hKP7rnKPISIrsYGxKhUOfjFUqgjbVbuLmD25ma8NWkp3JfCvnx7yyLJ0Wj4+clHgeOL+Abx05PNphql5UtVSxrmIdayvXUtpQikMcOG1OnHYncfY4nDYncba44LrTbr1vX4ZdbrPOFyy3Ofvlvo4meqUirMndxN3L7+a9be8B8O1DrmL9+pmsKq0H4MKjR/Pzs48kLaE/n4NQ4fL6vWyq3cS6ynWsrVjLusp1lDeV935gP2lP+HH2uOBFov39I3MeYWxq37uYa6JXqh8YY1i8YTGPrH4En/FRNHw6U1038Nh7u2nz+hmR6uLBCwuYc0T/j8ipOqtrrWNd5TorsVeuZX3V+r1mJkt0JDI5ZzKFOYUckXkEYM1/7PF7gkuPz4Pb796rPHTp9ofs217ud+PxeTrvEziX17/vBzv/dt7fyEvL6/PfrIleqX5UvKuY25bdRlVLFTkJOdxceD//+0/4dFsdAJdOH8Od/zGJFJfW7vuD3/j5uu7rTrX10obSvfYbmzKWKcOmUJhTSGFOIYemH4rddnBHTG2Pt7uLSPv7CWkTiLP3/T6PJnql+lllcyW3/utW1lSswSEOfjLtJporZ/LIu1/h9vnJTU/gV9+ezKxDs6Md6qDX6G6kpLIkWFsvqSyh0dPYaZ94ezz52fkU5hQyJWcKk3Mmk5WQFaWIDw5N9EodBB6/h9+u+S1Pf/40AKeNO43vHnYbd722mc/KrLb7q2aM4/YzJ5IUr9M1h8MYw9aGrcGkvq5yHZtrN2O6DMwxMmmkldSHTWFKzhQOzzwcp21o/YLSRK/UQfTe1ve4a/ld7PHsIS81j4dn/4Z/rIXfLv0Kj8/gctqYnpfJrEOzOf7QbI4cmaqzWgU0e5r5vPrzTs0wdW11nfZx2BwcmXVksLZemFPI8CTt5aSJXqmDrLS+lJs+uInNdZtJcCRw73H3kuc6nrv/up7VWzuP0p2R6OS4Q7KYdWg2sw7JZlxWYswPq2CMocXbQnVLNSVVJcHa+pc1X+IznUd0zXJlBWvqU4ZNYVLWJOLtQ3Nojn3RRK9UFDR7mrn/4/t5Y8sbAFw+8XJuLbqVumY/K76uYsXmaj7aXEV5XefeILnpCRx/aDazDstm5iFZZCcPrKRmjKHV10qTu4lGT6O1dDcG10PLmzxNNLgbguuN7kaaPNY+XRM6gF3sHJ5xeLAZpjCnkNxkHS00HJrolYoSYwwvf/kyD656EK/fy+ScycyfOr/joRkDuxvaKClv4LOyej4rq6extT0BWtvzspOYOiaDqWMzKMhNJzHOgSCICIJYcyC0/0+k83qX99ZZrWWLt6Vzgg5JxI3uRmu9m/ImdxNecwBzPwQkOBJIiUthYubEYBNMfnY+iU6dr3d/aKJXKspKKku4+V83s2vPrmiHEhHx9niSncmkxKWQ7EwmOc5a7/TemdJR3r4eWCbHJQ+5m6X97UBnmFJKHaCCnAJenvsyv1n9G0rrSzGB/1n/NxhjgmXtlS+Dwef30+L20uT2sqfNS5vXF9LjxGATSHDacMXZcTltOO3Sca4u5wz9PLBq1F0TcmjyTonbO1G3l+9PP28VPZrolTpIMlwZ3D/r/gM6R32Lh4+3VLN8cxUfba5iS+UemkK2ZyfHM+vQwI3dQ7N1YnMFaNONUoPazvoWlm/uSPyVjZ3Hxh+fncSsQ7M4/tBsjpuQTVqiNpfEKm2jV2oIMMawuaKJjzZXsXxzFR9vqaGpreOmqQgU5KYFu3HmZSeSluAkOd6hvVpigCZ6pYYgr8/PurL6YG3/0221eHx7//fusAlpCU7SEp2kJThJT3CSnhhnlSU4SU8MvBLiSG1/H9jmsA/EWY+HpgNO9CJyBvAoYAeeNMY82GX7zcA1gBeoBL5njNka2OYDSgK7bjPGnEMvNNErFXnNbi///qaG5Zur+Pc3NVQ0tlHX7KHFs3d/9nAlxzs6XQysi0NccD09ob08rtN+CU67/oqIsANK9CJiBzYBpwJlWFMCXmaM2RCyz0nAJ8aYZhG5HphjjLkksK3JGJPcl4A10St18LR5fdS3eKhv9lAXsqxrdlvlLR7qgtussrpA+f42CMTZbaQmOMlIdDImM5FxWYnkZSUFl7kZCTj110KfHGj3ymOAzcaYLYGTvQicCwQTvTHm/ZD9Pwau3P9wlVIHU7zDzrAUO8NSXH06zu83NLZ5AxcGN3XNno6LQHPX9+3rbmqbPbi9fqqa2qhqauOriqa9zm23CaMzEhiXlUReIPnnZScyLiuJ0RkJxDsO/vDCg1k4iT4X2B7yvgw4dh/7fx94K+S9S0SKsZp1HjTG/F+fo1RKDTi29rb9BCdj6dvTrK0e61dEVVMb22ta2Fq9h9LqPZRWNbO1eg876lvZWt3M1upmlnX9XIFR6QmdfgGMy0okLzuJsZmJuJx6Eegqov3oReRKoAg4MaR4nDGmXEQmAEtFpMQY83U3x14LXAswdmzfp9FSSg0eLqcdl9PO8FQXR41K22t7q8fH9ppmSqubgxeBrdXNlFbvoby2hbLA66PNe597ZJor5AJg/SIYF7gYDNXhocP5q8uBMSHvRwfKOhGRbwF3AicaY4KdeY0x5YHlFhH5AJgK7JXojTELgYVgtdGH/ycopWKNy2nnsOEpHDY8Za9tbq+fstrmYOIPXW6vaWZnfSs761v5eEvNXscOS4nv9Aug/YIwOiOBVJczZoeLDifRrwIOE5HxWAn+UuDy0B1EZCrwBHCGMaYipDwDaDbGtIlINjAL+FWkgldKDT1xDhsTcpKZkLN3Hw+vz8+OutZA4t8T8ougmW3VzVQ0tlHR2Ma/S/e+CIDViyjF1f5ydlk6SA1ZT4nvvD3V5STZ5cA+AC8WvSZ6Y4xXRG4A3sHqXrnIGPO5iNwHFBtjXgceApKBvwS6TLV3o5wEPCEifsCG1Ua/odsPUkqpA+Sw2xiblcjYrEQgp9M2n9+wq6GVrVV79moSKq9tocntpanNeu2s3/8YkuLsnS4OoReM1DAuIv3xy0IfmFJKKaxeRE1uL42tXhpbPZ2WDd2UdV1vaPXQ1Obd7y6n7d6cfwJHjkrt83E6eqVSSvXCZhNSXU5SXU5g/waDi8TFIsUV+bSsiV4ppSIkEheL/qCPnimlVIzTRK+UUjFOE71SSsU4TfRKKRXjNNErpVSM00SvlFIxThO9UkrFOE30SikV4wbkEAgiUgls3c/Ds4GqCIYzmOl30Zl+H53p99EhFr6LccaYnO42DMhEfyBEpLin8R6GGv0uOtPvozP9PjrE+nehTTdKKRXjNNErpVSMi8VEvzDaAQwg+l10pt9HZ/p9dIjp7yLm2uiVUkp1Fos1eqWUUiE00SulVIyLmUQvImeIyJcisllEbo92PNEkImNE5H0R2SAin4vIj6MdU7SJiF1EPhWRN6IdS7SJSLqIvCIiG0XkCxE5LtoxRZOI3BT472S9iLwgIq5oxxRpMZHoRcQOPAacCRwJXCYiR0Y3qqjyArcYY44EZgA/GuLfB8CPgS+iHcQA8SjwtjFmIlDIEP5eRCQXmA8UGWPyATtwaXSjiryYSPTAMcBmY8wWY4wbeBE4N8oxRY0xZqcxZk1gvRHrP+Tc6EYVPSIyGvgP4MloxxJtIpIGzAb+F8AY4zbG1EU3qqhzAAki4gASgR1RjifiYiXR5wLbQ96XMYQTWygRyQOmAp9EN5Ko+h/gp4A/2oEMAOOBSuCpQFPWkyKSFO2gosUYUw48DGwDdgL1xph/RDeqyIuVRK+6ISLJwBLgJ8aYhmjHEw0iMheoMMasjnYsA4QDOBp43BgzFdgDDNl7WiKSgfXrfzwwCkgSkSujG1XkxUqiLwfGhLwfHSgbskTEiZXknzPGvBrteKJoFnCOiJRiNemdLCJ/jm5IUVUGlBlj2n/hvYKV+IeqbwHfGGMqjTEe4FVgZpRjirhYSfSrgMNEZLyIxGHdTHk9yjFFjYgIVhvsF8aY30Q7nmgyxvynMWa0MSYP69+LpcaYmKuxhcsYswvYLiJHBIpOATZEMaRo2wbMEJHEwH83pxCDN6cd0Q4gEowxXhG5AXgH6675ImPM51EOK5pmAVcBJSKyNlB2hzHmzSjGpAaOG4HnApWiLcC8KMcTNcaYT0TkFWANVm+1T4nB4RB0CASllIpxsdJ0o5RSqgea6JVSKsZpoldKqRiniV4ppWKcJnqllIpxmuiVUirGaaJXSqkY9/8BXW6b/7qb9kYAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "pd.DataFrame(history.history).plot(lw=2);" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXxU1f3/8deZJZksk32DhEDYkV3ZBBEEUUpVbC2iVqu4tbUu9duv1lrb+mtta2u/bbXFra7UBa2KdUFQFEQUlLBvshgCSUjISpJJMsks5/fHnUz2EMgkk4TP8/G4j7nLmTtnQnhzOPfcc5XWGiGEEL2fKdgVEEIIERgS6EII0UdIoAshRB8hgS6EEH2EBLoQQvQRlmB9cEJCgh40aFCwPl4IIXqlLVu2FGutE1s7FrRAHzRoEJmZmcH6eCGE6JWUUkfaOiZdLkII0UdIoAshRB8hgS6EEH2EBLoQQvQREuhCCNFHSKALIUQfIYEuhBB9RNDGoQshRI/n9YLbCa4acNeAywmu6kb7fK+Nj7f12rhMxvlwwS8CXl0JdCFE7+RxG+Hqqoa6Kl+wtrVeBXXVDevNg7l5KNcfczu7pu6Rrd7o2WkS6EKIwNLaCMfaSqhzNH2tdUBdpa/l2mjxb9caoequ9QVsbevlXNXgqeue72OxGYs1HKw2sISB1bdYbI32NX9tXKbZqz2la6raJWcVQvQsXo8vUCuNVqunFtx1Rih6asHjMsLTU9ewtHe8PphrGwd2/XolaG83fCkFIRG+oA1rWA8J9+2rX49oetwa1hDO1vD2w9piA1PvudQogS5ET1TfyvW3cKsaQrOuWYjWVTZq/ToaBXejkHXXdG/9LTYIiYTQSAixQ6jdt+7bZw0HS6gRoJbQRi3dRot/u1G55i1dpbr3e/VwEuhCnA6P+yQXwer7ZJ2tv/rD2BfUjcO6fl9AW7nKF6p2o6VqCQVzKJhDwGz1bYc0LJaQpttN9oUa5wiN9J3P3mjd92q2BrDuoqMk0MWZyeUEZ3mz5YRvabSv5kQr5crB6+r6OjZp5TZq3YZEtNLqtTcN1OYtY2u4tGbPAB0KdKXUfOBRwAw8o7V+uNnxgcBzQCJQClyrtc4NcF2FaEprozVbU9ZsKW22faJlmc6OXlCmNi6E2Wj1Ipj/1ddX6w/lyFbWfSFtlvaWODUn/Y1RSpmBpcA8IBfYrJR6R2u9t1GxvwDLtNYvKqXmAH8EruuKCos+yOU0WsY1Jxpea8ra31cfzKfbUjZZISwGbNFgq39ttIQ139e4bJTR9SAtXtHDdKQJMAU4pLXOAlBKLQcWAo0D/Szgf3zra4G3A1lJ0ctobVyIqyyAyvyWr47CpuHcmQt21nAIi220xEBYXLN9viXct98WY7SYJZBFH9ORQE8Fchpt5wJTm5XZAXwXo1vmO4BdKRWvtS5pXEgpdStwK0B6evrp1lkEk8sJFXmtBHVB021XVcfP6W8txzR6jW1lX0xDINeHtNXWdd9ViF4mUJ10/wv8Uyl1A7AeyAM8zQtprZ8GngaYNGmSDtBni0By10J5Lpw4AieOQpnvtX5xFHTsPNZw4+YJe7+Wr5FJvla0L6hDIqS1LEQAdCTQ84ABjbbTfPv8tNbHMFroKKUigSu01icCVUkRQF4PlGU3CulmgV2Z3/77lRmiUiGqf9uBbU8xRldISAvRrToS6JuBYUqpDIwgvwq4pnEBpVQCUKq19gK/wBjxIoLNXQuFeyF/JxTshPwdcHyPcdt0W5QZolMhZiDEpDdafNv2fjL6Qpwy7fWi3W5wudAeD9rtNrbdbv92w7oH3C7/una7oPF7PB5fY0EZryaFUsp3R2ejbaWM0UhKoUxNt7XbZXye2412udCu+vW6hnq63cb++nW3C+3yvc/rBY1xvUhrjA3Q9duNj/mO60b7wyZMIO7a7wf853zSv5laa7dS6nZgNcawxee01nuUUr8FMrXW7wCzgT8qpTRGl8tPAl5T0b7aSijYbYR2wU4jxIv2gdfdsmxUKsRmNA3s2PrA7i+BfYbRWuOtqsZbWYGnogJvhfHqqajEW1GOp6KyyX5vZSVeVx243I3C2WVsu91Nw7p+3dsdUwH0Il4PBCPQAbTWK4GVzfb9utH6G8Abga2aaFNVcdPgzt8BpVnUtxIaKEgYDinjoN843+t4Y7SHCArt8eA5cQJvZSWeSgdeRyUehwNvi3UHHkdlq+ve6mowmVBmM1gsKIsFZTajLBZj279uRpktLddNJuOclRV4yyvwVFYard6uZrU2ravVYtSp1bo3fC8sZpTF2rBe3+DwevG3fL2+lrDXi6bRttagvS3KYLUY56yvk8WCslp9+y0oa4h/n/HqO24x3tfkfwH1/1Oo72FUrR1TvlVj3Zqa2iU/YmmK9QZaG10n+96Ffe/B8V0ty5iskHxWQ2injIPk0cbNKqKhFVpRjre62viLarOhQkIw2Wyo0FDUaU7CpD0ePGVluIuLcReX4CkxXt3FxbhLivH410vwlJUFprXq8aA9Hqira/HP+OlQYWGYo6IwR9kxRUVjttsxR0c1XbdHGa+RdlSI1ReIFn8g+kPYv+07bjaD2WyEmehSEug9ldcLeZmw7x0jxMsONxyzhjeEdn3LO3GkMddGH6W1Rjudvtaqw2jhllf4uwladg806yroQCvUH/K2UEyhRsibQkONfaEhxj6bDWU2GwFeYgT1KYW0UphjYjBFR2GOiMRkt2OKjMQcWb8egdluxxQRickeaaxHRjYtEx4OWhvdHS43eOq7Nk7S91zfR+3x+M5lBLTZbkeF9N3fnTOJBHpP4nFB9mdGS/zrlU2HCIYnwMgFMPJSGDzLmEypF/DW1RndBA4H3qoqI5CrqvA6qhr2VRmvHofD2F9Vf6xR2aqqTncLqPBwX1hGGBe6nE68dXVopxNdW+u7OOaCysqWY25PwhwTgyUxAXN8Apb4eCwJCZgT4rEkJGJJiMcSH485IQFLXJzReg0AZbFAaO/4PRDdQwI92Oqq4ZuPjRA/sMqY+KledDqMugRGXgLp08Bk7rZqeWtrfS1bX99uZWVDP2+lo0no+sO4qrpJeHuqqsAVuEmsVGio0VqNiGi9e6C+S8BuxxwV7TseZZQ9SStUe73oujp0bS1eZy261om3Puib7cPtxhwbizneF9hxsUb/qhBBJoEeDDVlcGC1EeKHPm5663viSBh1qRHi/cYHbCy31hpXbi7O3bup/SYLT0V504CurGy48FZZabRUA8FqxRwR4Q9i4zXc+C9/RKN9kRFGUDcp5wtv37GuDE1lMqFsNrDZMEd32ccI0aUk0LtT/k74/O+w979NhxOmnuML8UshYWinP0ZrjbuggJrdu3Hu2o1z925q9uzBW15+8jf7KKvV6K+1R2KOtButXnskpsj6fb7A9QdwRENwNw5l6ZsVottIoHc1reHI57Dhb3BojbFPmY2nfo+6DEYsMG7k6QRXYSHO3Xt8wb0b5+49eEpKWpQzx8VhGzsG24iRRpdB44C22zFF2o1uCrsdk/TNCtHrSKB3Fa8XDnxgBHnuZmOfNQLOuQHOvQ2i0075lFpr3IVF1O7/2mh9+0LcXVjYoqw5OhrbmDG+ZTRhY8ZgSUmRoWNC9GES6IHmroPdb8CGv0PxfmNfWBxM/RFMuaXDN/V4ysupPXgQ54ED1B486FsOtdptYoqMxDZmDGFjRvtD3JqaKuEtxBlGAj1Qah2wdRlsXAoVvoc1RaXB9Dvg7OuMGQVb4a2pofbQNw2h7Qvw1lrdAKboaGzDhvmDO2zMaKzp6ad9U4wQou+QQO+s6lL48in46ilj9AoYI1Vm/BTGfg/MVrTHg/vYMVx5edTl5lF39Ig/wF1Hc3yT9zSlbDZChw4ldPhwQocN8y+WpERpeQshWiWBfrpO5Bit8a0vouuqcTtNuMLH4Uq5GJcnnro39uB69ENcuXm48vPB3cokWQAWC6EZg5qEdujw4VjT0qTVLYQ4JRLop0BrjXPjx1S//jfqDu3GVWXC5YjAVROLdmugGHi51feaExMI6Z+KNS0Na1qaP7xDMgbJ0D4hREBIoJ+E1hrnzp1UrFpN5aoPcOXX344f3rgU5pgYI6xTU7GmpWJNTSXEF97W/v0x2eRRaUKIriWB3gp/iH+wiooPV+M+1vAUH0uYh8ih4YTOvQHr8PFGgKemYo5s/aKnEEJ0Fwl0H601zh07qFi1umWIJyVhHxZClG07YYPiULe+3+mbgYQQItDO6ED3h/gHq6j48EPc+Y1CPDkZ+8UXETV/PmFVn6E+/jVYwuCaVyXMhRA90hkX6FprarZvp3LV6pYhnpJC1MUXYb94PmETxhujTL5eCe/8xijwnSch9ewg1VwIIdp3xgV68T/+QfHjT/i3/SE+fz5h48c3HSqYvxPevBnQMOcBGH1591dYCCE66IwKdG9tLaXL/g1A7HXXEbXgWy1DvF5lAbx6NbiqYNximPm/3VxbIYQ4NWdUoDvWrsXrcGAbPZqUX97fdkFXDSy/xriFf8BUuOwfAZuXXAghusoZdSti+TvvAhB92aVtF/J64e0fQ94WiEmHxS/3mse9CSHObB0KdKXUfKXUfqXUIaXUfa0cT1dKrVVKbVNK7VRKLQh8VTvHXVaGY/16MJmIWtBO9T59GPasgBA7XPM6RCZ2XyWFEKITThroSikzsBT4FnAWcLVS6qxmxR4AXtdaTwSuAh4PdEU7q3LVKnC7iZgxA0tiGyG98z/w6Z9AmWDR85A0qnsrKYQQndCRFvoU4JDWOktrXQcsBxY2K6OBKN96NHAscFUMjJN2t+R8Bf/9ibF+8R9h2LxuqpkQQgRGRwI9FchptJ3r29fYg8C1SqlcYCVwR2snUkrdqpTKVEplFhUVnUZ1T0/d0aPUbNuGCg/HPnduywInjhoXQT21MOkmmPrDbqubEEIESqAuil4NvKC1TgMWAP9WSrU4t9b6aa31JK31pMS2uj26QPm7RuvcfuFcTOHhTQ86K+CVxVBVBINnw7f+JCNahBC9UkcCPQ8Y0Gg7zbevsZuA1wG01hsBG5AQiAp2ltaaivrulksva3rQ6zFuHCrcCwnDYdGLYLYGoZZCCNF5HQn0zcAwpVSGUioE46LnO83KHAXmAiilRmEEevf1qbTDuWsXdUeOYE5MIOLcaU0PfvgrOLgawmLh6uUQFhOcSgohRACcNNC11m7gdmA1sA9jNMsepdRvlVL1Td6fAbcopXYArwI3aN3Kc9WCwH8xdMG3UZZG91FlPg+bloLJCotfgvghQaqhEEIERofuFNVar8S42Nl4368bre8FZgS2ap2nXS4qVhrVjmo8uiVrHaz03cp/6d9h0HndXzkhhAiwPn2nqOPzz/GUlhIyZAi2s3xD54sPwus/AK8bZtwFE68NbiWFECJA+nSgN1wMvRRVP3Llo9+AsxxGfBvmPhi8ygkhRID12UD3OBxUfvwxANGXXuLb6YLDnxrrCx6B1mZZFEKIXqrPJlrlR2vQtbWET5qENdV3H1RuJtQ5IGGEPHVICNHn9NlAr3jXGFnZ4mIoGDcQCSFEH9MnA911/DhVGzehrFaiLr644UDWWuN18OxgVEsIIbpUnwz0ivfeB62JnD0bc3S0sdNZYXS5KLMMUxRC9El9MtDr526JXtjoVv/sDaA9kDYJbFFtvFMIIXqvPhfozv0HqP36a0zR0UScf37DAX//+QVBqZcQQnS1PhfoFe8ZrfOo+fMxhYQ0HJD+cyFEH9enAl17vZS/+x7Q7EEW5XlQfABCIo0uFyGE6IP6VKBXf7UZd0EB1tRUwiZObDhQfzPRoPNkelwhRJ/VpwK9vH7s+aWXoBrfBfqNdLcIIfq+PhPoXqeTytUfAhB9WaPRLVrLBVEhxBmhzwS6Y906vA4HtjFjCB08uOFA4V6oKgR7P0gcEbwKCiFEF+szgV7+X6O7pcnFUGh6u788K1QI0Yf1iUB3l5Xh+OwzMJuJWrCg6UGZv0UIcYboE4Fe8cEH4HYTMX06loRGz6Z210H258b64NnBqJoQQnSbvhHo9Q+yaHwxFCB3M7iqIHEU2FOCUDMhhOg+vT7Q644epWb7dlR4OPa5c5oelLtDhRBnkF4f6OW+1nnUvAsxhYc3PVjffz5EhisKIfo+S0cKKaXmA48CZuAZrfXDzY7/DahPzXAgSWsdE8iKtkZr3ehmombdLTUnIG8LmCwwcHpXV0WIXs/lcpGbm4vT6Qx2VQRgs9lIS0vDau343e0nDXSllBlYCswDcoHNSql3tNZ768tore9uVP4OYGKLE3UB586duI4cxZyYQMS0qU0PZm8A7YUB0yDU3h3VEaJXy83NxW63M2jQoIaHqoug0FpTUlJCbm4uGRkZHX5fR7pcpgCHtNZZWus6YDmwsJ3yVwOvdrgGnVDf3RL97UtQlmb/NslwRSFOidPpJD4+XsK8B1BKER8ff8r/W+pIoKcCOY22c337WqvEQCAD+KSN47cqpTKVUplFRUWnVNHmtMtFxcqVQCs3E0HDBVHpPxeiwyTMe47T+bMI9EXRq4A3tNae1g5qrZ/WWk/SWk9KTEzs1Ac5NmzAU1ZGyNAhhI4a1fTgiRwoOQQhduh/dqc+RwgheouOBHoeMKDRdppvX2uuopu6WyrqHzN36WUt/yWr727JmAnmDl33FUL0AJGRkcGuQq/WkUDfDAxTSmUopUIwQvud5oWUUiOBWGBjYKvYksfhoPJjo1cn+pJvtywgsysKIc5AJ22+aq3dSqnbgdUYwxaf01rvUUr9FsjUWteH+1XAcq217rrqGio//AhdW0v45MlYU5t153u9ckFUiE4adN/7XXLe7IdbaYC1QmvNvffeywcffIBSigceeIDFixeTn5/P4sWLqaiowO1288QTTzB9+nRuuukmMjMzUUpx4403cvfdd5/8Q/qgDvVHaK1XAiub7ft1s+0HA1et9vnHnrd2MbRwD1QXQ1QqJAzrrioJIQLorbfeYvv27ezYsYPi4mImT57M+eefzyuvvMLFF1/ML3/5SzweD9XV1Wzfvp28vDx2794NwIkTJ4Jc++DpdR3MroICqjd9ibJaibr44pYFZLpcITqtoy3prrJhwwauvvpqzGYzycnJzJo1i82bNzN58mRuvPFGXC4Xl19+ORMmTGDw4MFkZWVxxx138O1vf5uLLrooqHUPpl5363/uG2+D1kRecAHmqKiWBfyPm5P+cyH6mvPPP5/169eTmprKDTfcwLJly4iNjWXHjh3Mnj2bJ598kptvvjnY1QyaXhXobo+XQ6+8AcD6gee0UqAWjnxhrA+e1Y01E0IE0syZM3nttdfweDwUFRWxfv16pkyZwpEjR0hOTuaWW27h5ptvZuvWrRQXF+P1erniiit46KGH2Lp1a7CrHzS9qsvFYjYR8uAfePnxl1h+PJqwbbl8Z2JaQ4GcL8FdA0mjITIpeBUVQnTKd77zHTZu3Mj48eNRSvHnP/+ZlJQUXnzxRR555BGsViuRkZEsW7aMvLw8lixZgtfrBeCPf/xjkGsfPL0q0AFmXTSFwxGJuN/dy71v7KR/dBhTB8cbB2V2RSF6NYfDARh3ST7yyCM88sgjTY5ff/31XH/99S3edya3yhvrVV0u9W6YkcEN0wfh8mh++NIWsoqMXwIZriiEOJP1ykAH+NUlZzF3ZBInql3c+MJmykoK4dg2MFllulwhxBmp1wa62aR47OqJjO4fRXZJNc/8+0XfdLlTISQi2NUTQohu12sDHSAi1MKz108mJcpGv2JjxgEto1uEEGeoXhnodZ46/3pKtI1nb5jETPMeAF4tGRqsagkhRFD1ukBfcXAFV7xzBcerjvv3jQ47wUBVQIUO51dfWXhzS24QayiEEMHRqwLd7XWzfP9ysiuyWbJ6CfmOfOOAb3RLWfI0PJi5762dbMoqCV5FhRAiCHpVoFtMFp6e9zRnxZ9FTmUOS1YvIc+R5w/0gZMWsGSGbzjjv7fwTf1wRiGEaMTtdge7Cl2i191YFB0azb8u+hc/+uhH7CrexZJVS3j26DfGEziGzOGBSYPJKa1mzb5CbnxhMytum0FcREiwqy1E7/JgdBedt/ykRS6//HJycnJwOp3cdddd3HrrraxatYr7778fj8dDQkICH3/8MQ6HgzvuuMM/be5vfvMbrrjiCiIjI/03KL3xxhu89957vPDCC9xwww3YbDa2bdvGjBkzuOqqq7jrrrtwOp2EhYXx/PPPM2LECDweDz//+c9ZtWoVJpOJW265hdGjR/PYY4/x9ttvA/DRRx/x+OOPs2LFiq75OZ2mXhfoAFEhUTw17yl+tOZH7CzayZKYEJ6zDCA9bjBmpXj0qolc+dRG9hyr4NZlmbx081RsVnOwqy2E6IDnnnuOuLg4ampqmDx5MgsXLuSWW25h/fr1ZGRkUFpaCsDvfvc7oqOj2bVrFwBlZWUnPXdubi5ffPEFZrOZiooKPvvsMywWC2vWrOH+++/nzTff5OmnnyY7O5vt27djsVgoLS0lNjaW2267jaKiIhITE3n++ee58cYbu/TncDp6ZaAD2EPsPHXhU9y24nK2OY+zJDaEZyuOMCh6EBGhFp67YTKXL/2czCNl3PvGTh69aoI8AFeIjupAS7qrPPbYY/6Wb05ODk8//TTnn38+GRkZAMTFxQGwZs0ali9f7n9fbGzsSc+9aNEizGajcVdeXs7111/PwYMHUUrhcrn85/3Rj36ExWJp8nnXXXcdL730EkuWLGHjxo0sW7YsQN84cHpVH3pzkSGRPOkM55waJ4W6jiWrl5BVngVAcpSNZ6+fTESImXd2HONvHx0Icm2FECezbt061qxZw8aNG9mxYwcTJ05kwoQJp3SOxg03p9PZ5FhERMNNh7/61a+44IIL2L17N++++26Lss0tWbKEl156iVdffZVFixb5A78n6dWBjstJeM6XPH68iCmJEyiuKebGVTdyqOwQAGf1j+Kf15yNScFjnxziDRnOKESPVl5eTmxsLOHh4Xz99dds2rQJp9PJ+vXrOXz4MIC/y2XevHksXbrU/976Lpfk5GT27duH1+ttt4+7vLycVN8jLF944QX//nnz5vHUU0/5L5zWf17//v3p378/Dz30EEuWLAnclw6g3h3oOZvA7SQ8eQz/vOhppvWbRomzhJs+vIn9pfsBuGBkEv/vstEA/OKtnWz8RoYzCtFTzZ8/H7fbzahRo7jvvvuYNm0aiYmJPP3003z3u99l/PjxLF68GIAHHniAsrIyxowZw/jx41m71ni4zcMPP8wll1zC9OnT6devX5ufde+99/KLX/yCiRMnNhn1cvPNN5Oens64ceMYP348r7zyiv/Y97//fQYMGMCoUaO66CfQOaobnuncqkmTJunMzMzOnWTNg7DhbzD9DrjoIZxuJz9d91M+z/ucmNAY/nXRvxgZNxKA3723l2c3HCbKZuGt22YwNCmy819CiD5k3759PTaoeorbb7+diRMnctNNN3XL57X2Z6KU2qK1ntRa+d7dQm/2uDmbxcajFzzK+Wnnc6L2BDetvom9JXsBuH/BKC4clUyF082NL2ymsKL9/jIhhGjsnHPOYefOnVx77bXBrkqbem+gV5dC/g4wh0D6uf7doeZQ/jb7b8weMJuKugpu/vBmdhfv9s3OOIGxqdEcLa3myqc2kltWHcQvIIToTbZs2cL69esJDQ0NdlXa1KFAV0rNV0rtV0odUkrd10aZK5VSe5VSe5RSr7RWJqAOrwe0b7rc8CaHQswh/HXWX5mbPpfKukpu+fAWdhTtIDzEwos3TmFMqjHl7qInN8rdpEKIPuOkga6UMgNLgW8BZwFXK6XOalZmGPALYIbWejTw0y6oa1NZvu6WNh43ZzVbeWTWI1w08CIcLgc//OiHbCvcRlxECK/cMo1JA2PJL3dy5ZMb2XusosurK4QQXa0jLfQpwCGtdZbWug5YDixsVuYWYKnWugxAa10Y2Gq2ogOPm7OarPzp/D/xrUHfospVxQ8/+iGZBZlE2awsu2kKM4clUFJVx1VPb2TLkZPfZSaEED1ZRwI9FchptJ3r29fYcGC4UupzpdQmpdT81k6klLpVKZWplMosKio6vRoDlB6GsmywxUC/9m86sJgs/GHmH7hk8CXUuGu47ePb2FywmfAQC89cP4mLRxsXSq979ks+P1R8+nUSQoggC9RFUQswDJgNXA38SykV07yQ1vpprfUkrfWkxMTE0/+0+tZ5xvlgOvkcLRaThYdmPMRlQy4zQn3Nbbx96G1CzCaWXnM2352YSnWdhyUvbGbN3uMnPZ8QIvgiI9seepydnc2YMWO6sTY9Q0cCPQ+MyQx90nz7GssF3tFau7TWh4EDGAHfNU7Sf94as8nM72b8ju8N/x5Oj5Nfff4r7l1/L9UeB39ZNJ7rpg2kzu3lhy9t4b/bm389IYTo+ToyGcFmYJhSKgMjyK8CrmlW5m2MlvnzSqkEjC6YrEBW1M/r8Y1wod3+89aYlIlfT/s1E5Mm8vtNv2dV9ip2FO3g4ZkP89uFE4m0WXhi3Tf89LXtVNd5uHpKesCrL0RvMPbFsV1y3l3X72rz2H333ceAAQP4yU9+AsCDDz6IxWJh7dq1lJWV4XK5eOihh1i4sPklvPY5nU5+/OMfk5mZicVi4a9//SsXXHABe/bsYcmSJdTV1eH1ennzzTfp378/V155Jbm5uXg8Hn71q1/570ztDU7aQtdau4HbgdXAPuB1rfUepdRvlVKX+YqtBkqUUnuBtcA9Wuuuuce+YCfUlEFMOsRmnPLblVJcNuQy/nPpfxgTP4b8qnyWrF7C4zse52cXDeWei0egNfzirV0881nX/JskhGhp8eLFvP766/7t119/neuvv54VK1awdetW1q5dy89+9jNO9e72pUuXopRi165dvPrqq1x//fU4nU6efPJJ7rrrLrZv305mZiZpaWmsWrWK/v37s2PHDnbv3s38+a1eDuyxOjRdmNZ6JbCy2b5fN1rXwP/4lq7V+O7QTkyHmx6VzrIFy3hi+xM8s+sZntzxJBuPbeThmQ8TGTqa37yzh4fe30el081PLxwmU++KM0p7LemuMnHiRAex7JAAACAASURBVAoLCzl27BhFRUXExsaSkpLC3Xffzfr16zGZTOTl5XH8+HFSUlI6fN4NGzZwxx13ADBy5EgGDhzIgQMHOPfcc/n9739Pbm4u3/3udxk2bBhjx47lZz/7GT//+c+55JJLmDlzZld93S7R++4U7cBwxY6ymqzcefadPHvxsySFJ7GjaAeL3l1EQsoe/rJoPCYFj358kIfe33fKrQIhxKlbtGgRb7zxBq+99hqLFy/m5ZdfpqioiC1btrB9+3aSk5NPOs1tR11zzTW88847hIWFsWDBAj755BOGDx/O1q1bGTt2LA888AC//e1vA/JZ3aX3BfqkJTD+GsiYFbBTTk6ZzFuXvcWF6RficDm477P72FK9lP9bPAKrWfHshsP84q1deLwS6kJ0pcWLF7N8+XLeeOMNFi1aRHl5OUlJSVitVtauXcuRI0dO+ZwzZ87k5ZdfBuDAgQMcPXqUESNGkJWVxeDBg7nzzjtZuHAhO3fu5NixY4SHh3Pttddyzz33sHXr1kB/xS7V82ZoP5nR3zGWAIsOjeavs//KWwff4k+b/8R7We+xLXIb93/nPv703xqWb87BUevmr1dOIMTS+/4dFKI3GD16NJWVlaSmptKvXz++//3vc+mllzJ27FgmTZrEyJEjT/mct912Gz/+8Y8ZO3YsFouFF154gdDQUF5//XX+/e9/Y7VaSUlJ4f7772fz5s3cc889mEwmrFYrTzzxRBd8y67Tu6fP7SKHyw/z8/U/Z1/pPszKzKXpN7Bi7QgctV7mjEzi8e+fLc8oFX2OTJ/b85xZ0+d2kYzoDF5a8BI3jL4Bj/bw9pFnGXn2y8TYHXzydSE3PP8VJ6rrgl1NIYRoQgK9DSHmEH426Wc8Ne8pEsMS2X9iB7ZBjxKXtJdNWaUsePQzvjpcGuxqCnFG27VrFxMmTGiyTJ06NdjVChrpcumAUmcpv/n8N6zLXQdApGs6+VkXYtLh3D5nGHfOGYrFLP82it5Nulx6Huly6QJxtjgem/MYD0x9gFBzKA7rF8SP/CuW2M957OOvuerpTfKwDCFE0Emgd5BSisUjF/PaJa8xJWUKddpBaPK7RA19lG0lG/nWo+t5b+exYFdTCHEGk0A/RUNihvDMRc/w6AWPkm5PR1sLCR/wAu7Ep7nzzQ/4+Rs7qa5zn/xEQggRYBLop0EpxZz0Oby98G3umXQPdqsdS+RBwjMe5e3cx1jwzw/YnVce7GoKIc4wEuidYDVb+cHoH/D+d9/nqhFXYTaZCIn9kuKY/8eiV//Ak+v345W7S4XoEu3Nh36mkkAPgFhbLL+c9kveuuxNpvebgTLXYklcyT/238rlLzxJYUVg5p4QQvQ8bnfP6WLtfbf+92BDYobw1EVP8nne5/xmwx85zhEO8zgXvrqaeyffw7Vnzwh2FYXokH0ju2b44qiv97V5LJDzoTscDhYuXNjq+5YtW8Zf/vIXlFKMGzeOf//73xw/fpwf/ehHZGUZU2Y/8cQT9O/fn0suuYTdu3cD8Je//AWHw8GDDz7I7NmzmTBhAhs2bODqq69m+PDhPPTQQ9TV1REfH8/LL79McnIyDoeDO+64g8zMTJRS/OY3v6G8vJydO3fy97//HYB//etf7N27l7/97W+d+tmCBHqXmJE6g1WL3ub5nctZun0pHts3PLzzx7x+YBZPfPuXpNo7PvWnEGeKxYsX89Of/tQf6K+//jqrV6/mzjvvJCoqiuLiYqZNm8Zll1120umsbTYbK1asaPG+vXv38tBDD/HFF1+QkJBAaalxc+Cdd97JrFmzWLFiBR6PB4fDQVlZ+w+Or6uro/5emrKyMjZt2oRSimeeeYY///nP/N///R+/+93viI6OZteuXf5yVquV3//+9zzyyCNYrVaef/55nnrqqc7++AAJ9C5jMVm4ZcK1fG/kpdyx8hG2V7zH4dp1fOvNjVwz4nrumHQjEdaIYFdTiFa115LuKoGcD11rzf3339/ifZ988gmLFi0iISEBgLi4OAA++eQTli1bBoDZbCY6Ovqkgd74SUa5ubksXryY/Px86urqyMgwHr6zZs0ali9f7i8XGxsLwJw5c3jvvfcYNWoULpeLsWMD84Qo6UPvYrG2aF767kM8Mu3fWJ3j0KqWlw88zazlF/LPbY9TXiujYYSoF6j50AMxj7rFYsHr9fq3m78/IqKhQXbHHXdw++23s2vXLp566qmTftbNN9/MCy+8wPPPP8+SJUtOqV7tkUDvJt8aOZZ117/A2dZf4K4eRK3XwVM7n2Duf+bx1y1/pbimONhVFCLoAjUfelvvmzNnDv/5z38oKTGekFnf5TJ37lz/VLkej4fy8nKSk5MpLCykpKSE2tpa3nvvvXY/LzU1FYAXX3zRv3/evHksXbrUv13f6p86dSo5OTm88sorXH311R398ZyUBHo3irJZefGaa3h8zrNEld2J2zGMWk8Nz+9+novfmM8fvvwD+Y78YFdTiKBpbT70zMxMxo4dy7Jlyzo8H3pb7xs9ejS//OUvmTVrFuPHj+d//sd4auajjz7K2rVrGTt2LOeccw579+7FarXy61//milTpjBv3rx2P/vBBx9k0aJFnHPOOf7uHIAHHniAsrIyxowZw/jx41m7dq3/2JVXXsmMGTP83TCBIJNzBYnT5eFf67NY+sVaiP0Yq30vABZl4dIhl3LT2JsYGDUwyLUUZxKZnKt7XXLJJdx9993MnTu3zTIyOVcvYbOauWPuMD76ybXMjLqHqqyf4iqfgNvrYcWhFVz29mXc++m9HCg7EOyqCiEC6MSJEwwfPpywsLB2w/x0dGiUi1JqPvAoYAae0Vo/3Oz4DcAjQJ5v1z+11s8EsJ591oC4cP71g0l88vUAHnxnMDlFRwmJ/5TQ2K18kP0BH2R/wOwBs7l17K2MTQzMlXAh+opdu3Zx3XXXNdkXGhrKl19+GaQanVxMTAwHDnRNQ+2kga6UMgNLgXlALrBZKfWO1npvs6Kvaa1v74I6nhHmjExm+pAEnvo0i8fXJVFZPJfIpA2YY75iXc461uWsY1q/adw67lYmJU866ThcIU6H1rpX/W6NHTuW7du3B7saXeJ0usM70uUyBTiktc7SWtcBy4GT36olTpnNauauC4fx0d2zmDN0OJXHLqF8/71E1lyEzRzOpvxN3Lj6Rn7wwQ9Ylb0Kl8cV7CqLPsRms1FSUnJaQSICS2tNSUkJNpvtlN530ouiSqnvAfO11jf7tq8DpjZujfu6XP4IFAEHgLu11jmtnOtW4FaA9PT0czo6BOlMtWbvcf7fe3vIKa0BUzUTRu+hyLSGijpj7Hq8LZ4rhl/BouGLSImQu09F57hcLnJzc095vLboGjabjbS0NKxWa5P97V0UDVSgxwMOrXWtUuqHwGKt9Zz2znumj3LpKKfLw+PrvuHJT7+hzu3FbvMwZ0oOR11rOHTiEAAmZeL8tPO5asRVnNv/XExKrnUL0Vd1NtDPBR7UWl/s2/4FgNb6j22UNwOlWuvo9s4rgX5qsourePDdPazbXwTAgLgwLp9WR4H+hDVHP8LtNWZ8G2AfwJXDr+TyoZcTY4sJZpWFEF2gs4FuwehGmYsximUzcI3Wek+jMv201vm+9e8AP9daT2vvvBLop05rzUd7j/Pwqq/JKqoCYHBCBDfNSqAqZCNvHPwP+VXGjUkhphDmZ8znyhFXMi5hXK+60CWEaFunAt13ggXA3zGGLT6ntf69Uuq3QKbW+h2l1B+BywA3UAr8WGv9dXvnlEA/fW6Pl/9uP8ajHx/kaKnxcOphSZHcdeEQImIO8fqB1/g873M0xp/tqLhRXDniShZkLCDcGh7MqgshOqnTgd4VJNA7z+Xx8tbWXB77+BB5J2oAGNUvirsvHMaoAS7+c/A/vH3wbcpqjfkjIq2RXDbkMhaPWMzgmMHBrLoQ4jRJoPdxdW4vr2XmsPSTQxT4no40Li2au+cN59whUXx05CNe3/8624saxuuOiR/DgsELmD9oPonhicGquhDiFEmgnyGcLg+vfnWUpWu/odhRC8DZ6TH8z7wRzBgaz4GyA7y2/zVWHl5JlcvogzcpE1NSprAgYwEXDrwQe4g9mF9BCHESEuhnmJo6D//elM2Tn2ZRWlUHwJSMOH42bzhTB8fjdDv5NPdTVmat5LO8z3B5jRuUQkwhzBowiwUZC5iZNpNQc2gwv4YQohUS6GcoR62bF7/I5un1WZTXGKF93tAE7p43nHMGGlN2lteWs+bIGlYeXsnmgs3+C6l2q50LB17IgsELmJw8GbPJHLTvIYRoIIF+hqtwunhuw2Ge/ewwlbXGePUpg+K48bwM5p2VjNlkDGksqCpgdfZq3s96n32lDY8gSwxLZH7GfL6d8W3Oij9LhkAKEUQS6AKAE9V1/OuzLJZ9ccQf7Olx4dwwfRBXTh5AZGjDXG1ZJ7JYeXgl72e9T64j179/UNQgf3/70JihEu5CdDMJdNFEpdPFfzJzef6Lw8Y8MYA91MLiyQO4fvogBsQ1jFXXWrOreBfvZ73PquxVlDpL/cfS7enMHTiXuelzGZswVqYcEKIbSKCLVnm8xp2nz31+mK8OG0FtUnDx6BRuOi+DcwbGNmmBu71uvsr/ilXZq1iXs84/vh0gKSyJC9IvYG76XCalTMJqsrb4PCFE50mgi5PalVvOc58f5t0dx3B7jd+J8WnR3HheBgvG9sNqbtr6dnvdbCvcxsdHP+bjox9TUFXgPxYVEsWstFnMTZ/L9NTphFnCuvW7CNGXSaCLDjte4WTZxmxe/vIoJ6qNkTH9om384NxBXD1lADHhIS3eo7Vmb+lePj5ihHtWeZb/mM1sY0bqDOamz+X8tPOJDm13zjYhxElIoItTVlPnYcW2PJ77/DCHCh0AhFnNXHFOKktmZDAkMbLN9x4uP8zHRz/mk6OfsKt4l3+/RVmYlDKJuelzmZU2i36R/br8ewjR10igi9Pm9WrWHyzi2Q2H+exgsX//jKHxLJ6czsWjkwm1tD1GvaCqgE+OfsInRz8h83gmHu3xHxsaM5TzUs9jZupMJiZNxGqWfnchTkYCXQTEgeOVPP/5YVZsy8Pp8gIQG27lu2encdXkAQxLbn/agBPOE3ya+ylrc9ayKX+Tf/oBgHBLOOf2P5fzUs/jvNTz5AlMQrRBAl0EVHmNi3e25/HqVznsza/w7z9nYCxXTR7AJeP6ExbS/p2lLo+LbYXb2JC3gc/yPvM/fanesNhhzEydyczUmYxPGi+jZoTwkUAXXUJrza68cpZvzuGd7cdw+G5WsodaWDixP1dNTmdMascuguY78vks7zM25G1gU/4matw1/mOR1kjO7X8uM1NnMiN1BknhSV3yfYToDSTQRZerqnXz/s58Xt18lG1HT/j3j0mN4qrJ6Syc0B+7rWOt7DpPHVsLt/JZrhHwjUfNAIyMG8nUlKlM6TeFs5POJjKk7Qu0QvQ1EuiiW+0vqGT55qO8tTXPPylYmNXMt8f14+opAzg7PfaUpgzIc+SxIdfomvky/0ucnoan0puVmbPiz2JyymSmpkxlQtIEeSqT6NMk0EVQOF0eVu8p4NWvjrIpq2HKgGFJkVxxThoLJ/SnX/Sp3XRU66ll6/GtbC7YzFcFX7GneA9u7fYft5gsjE0Yy+SUyUxJmcL4xPHYLLaAfSchgk0CXQTd4eIqlm8+yptbcil2GHO0KwXTh8Rz+YRUvjW2X5PJwTqq2lXN1sKtfFXwFZvzN7O3dC9e7fUfDzGFMC5xHFP6TWFKyhTGJYyT4ZGiV5NAFz2Gy+Nl7deFvL09jzV7C6nzGOFrs5q46KwUvnN2KjOHJmAxn95EX5V1lWw5vsUI+ILN7C/d75/jHYw7VyckTWBswlhGx49mdMJoksOTZdZI0WtIoIseqbzGxcpd+azYmsdX2Q1dMgmRIVw6vj/fnZjGmNSoToVteW05mQWZfFnwJZsLNrcYHgkQZ4tjdPxozoo/yx/yMpJG9FSdDnSl1HzgUcAMPKO1friNclcAbwCTtdbtprUEumgsp7Sa/27P461teWQVNdxwNDQpku9MTGXhhP6kxXb+YmdJTQnbCrexp2QPe0v2sqdkD+W15S3KJYQlGOHuC/iz4s8iISyh058vRGd1KtCVUmbgADAPyAU2A1drrfc2K2cH3gdCgNsl0MXp0FqzM7ecFdvyeHfHMUp8z0QFmJoRx3cmprJgXD+iOjgEsiOfl+fIY0/JHn/I7y3eS6WrskXZpPAkf8iPTRjL6ITRMtmY6HadDfRzgQe11hf7tn8BoLX+Y7Nyfwc+Au4B/lcCXXSWy+Pls4NFvLU1j4/2HqfWbfS3h1hMnD8skfljUpg3Kpno8MBe5NRak1OZ06QVv7dkb5OpCuoNihrE2ISxjE0cy7iEcQyPHS4XXUWX6mygfw+Yr7W+2bd9HTBVa317ozJnA7/UWl+hlFqHBLoIsAqni1W7ClixLY9Nh0uo/7W1mBTnDok3wv2sZJLsXTNE0au9HKk4wt6Svewu3s2u4l3sK9lHnbeuSbkQUwij4kcZIe8L+rTINLnoKgKmSwNdKWUCPgFu0FpntxfoSqlbgVsB0tPTzzly5Mjpfytxxjpe4eTDvcdZtTufTVmleHwP5FAKJg2MZf6Yflw8Ojkgfe7tcXlcHCg7wK7iXewq3sXOop1kV2S3KBdni2NMwhjGJhiteOmqEZ3RpV0uSqlo4BvA4XtLClAKXNZeK11a6CIQyqrq+GjfcVbvLuCzg8X+YZAA49KiuXh0Ct8ak8LgduZvD6Ty2nL2FO9hZ/FOf0u+8XNY66VGpjIsdhjDY4f7l3R7OmZT+5OaCdHZQLdgXBSdC+RhXBS9Rmu9p43y65AuFxEElU4Xa/cXsXp3AWv3F1Jd1zD3+vDkSOaPTmH+mH6M6mfvti6Q+ouu9S34XcW7+Lr0a2o9tS3K2sw2hsYMZXjc8CZBL6150Vgghi0uAP6OMWzxOa3175VSvwUytdbvNCu7Dgl0EWROl4f1B4pYtbuAj/Ydp9LZMD1Aelw4c0YmMXtEItMGx2Ozdm+r2O11c7TiKAfKDjRZ8qvyWy2fFJ7E8NjhjIgdwfDY4QyLHcagqEFy8fUMJTcWiTNandvLpqwSPthdwEd7C/xTD4Bxh+q5g+O5YGQSs4cnkR4fvIm9ymvLOVh20B/wB8sOcvDEwSZTCdczKzOpkalkRGf4l0FRg8iIziDWFhuE2ovuIoEuhI/Hq9l2tIx1+4tYu7+QPccqmhwfnBDB7BFG631KRly3t96b82ovOZU5DS350gMcPHGQPEdekzlrGosJjWkS8PXrafY0LKZTny9H9CwS6EK0obDCyacHili3v4j1B4uadM2EWc1MHxLP7JFJzB6eyIC4njMtb62nlqMVR8muyOZw+WH/kl2R3ep4eTBmoky3p/uDflD0IP+69NP3HhLoQnSA2+Nl69ETrNtfyLr9RU0erwcwJDGCC0YkMXtEEpMGxQa99d4arTVFNUVNAr5+va0+ejCGVvqDPmqQP+ylVd/zSKALcRqOVzj51Nc1s+FgMZW1Da33EIuJc9JjmT4knulD4xmXFoP1NGeI7C7VrmqOVh41gr482x/42RXZrfbTg9GqH2Af0CTsM6IzGGAfQJwtTm6YCgIJdCE6yeXxsuWI0fe+/kDL1nt4iJkpGXFGwA9JYFS/KMym3hF2Xu2lsLqwSYs+u9wI+vZa9eGWcNLsaaRFpjHAPoAB9gGk2Y31fhH9ZBROF5FAFyLASqvq+DKrhC++KeGLb4r5pqhpv3V0mJVpg+OYPiSB6UPiGZoU2Stbs/Wt+uzybA5XHPaHfW5lbqsTmNUzKRP9IvqRFplmhL69aehHhUR147foWyTQhehixyucbPSF+xfflJBb1rQLIyEy1Nd6j+fcIfGkx4X3yoBvrLy2nNzKXHIqc8h1+F592wVVBU0eLNJcpDWSfpH96BdhLCkRKf71/pH9SQhLkL77NkigC9HNckqrmwR8YWXTO0OTo0KZNCiOSQNjmTwojpEp9tN+SlNPVOep45jjWKthn+fIa7PPvp5ZmUkKT/KHff/I/k2CPyk8iaiQzj38pLeSQBciiLTWfFNUxUZfuG/KKqGs2tWkTESImbMHxnKOL+AnDIgh4jSesdobaK0pry0nvyq/YXEYrwVVBeRX5VNUU3TS84RZwkgKT2qyJIcnN1nviy19CXQhepD6gM/MLmVzdhmZR0o5UlLdpIzZpBjdP8of8JMGxpIU1TVTA/dEdZ46jlcf9wf8Mccx/3p+VT7Hq45T7a4+6XlMykS8Lb5F6KdEpBhLeArJEcmEmEO64VsFhgS6ED1cYaWTLdll/oDfc6zCPy1wvfS4cCYNMgJ+YnoMw5LsvWYkTVdw1DkorC7kePVxCqsL/euNt0tqStrty68XZ4vzB3x92Nd38aREpPSolr4EuhC9TFWtm+05J8j0BfzWI2VUNZo9EoyhkmNSo5kwIIYJA2IYPyCG/tG2M7JfuS0ur4uSmhJ/yB+vMl4LqgooqC6goKqAwupCPNrT7nlMykRiWKI/4BPDEokPiyfeFk9CWAIJYQnEh8UTZ4vr8uCXQBeil3N7vHxdUMnm7FK2HClje86JFiNpwBhNYwR8NOMHxDAuLYboMBkP3h6P10NxTbE/4Fss1QUdbukrFDGhMUbYh/nC3pbgX4+3GfuTw5OJscWcVn0l0IXog4odtezMPcH2nHK255xgR84JymtcLcoNToxgQloME9JjGJ8Ww8h+dkItPW/agp7M5XH5+/TrA76kpoTimmJKnMZrcU0xZc6yDgX/7AGz+cecf5xWXdoL9J7RKSSEOGUJkaHMGZnMnJHJgHGx9UhJNTtyT7Dt6Al25J5gz7EKsoqqyCqq4q1teQCEmE2M7GdndP9oxqYay/CUSAn5dljNVv8NUu1xe92cqD3hD/vGgd/4H4GB9oFdUk9poQvRh9W5vewvqGR7Thnbc8rZkXuCQ4WOFuWsZsXwZDtjU6MZ7Qv5kSn2HjkB2ZlOulyEEH6VThd7jlWwO6+c3Xnl7MorJ6u4iuZRYDYphiVFGq34tGjGpEYzKiWKsBAJ+WCSQBdCtMtR62ZffgW7co2Q332snEOFDrythPzQxEhG9bMzql+Uf0m0hwan4mcgCXQhxCmrrmsU8r4W/cFCR4vx8QAJkSGNAt7OyJQohiRGEmLpO9MZ9BQS6EKIgHC6POzLr+Drgkr25VcY6/mVTeaKr2c1K4Ym2RmVYm8S9vGR0prvDAl0IUSX0VqTW1bjC/hKX+BXkF3S+q35ifZQRqbYGZZkZ3hyJMNT7AxLisRuk/HyHSGBLoTodlW1br4uqOTrggp/2H+dX9Hijtd6qTFhDEuOZESynWHJRtgPTYokPERGVzfW6UBXSs0HHgXMwDNa64ebHf8R8BPAAziAW7XWe9s7pwS6EGcer9dozR84Xsn+45UcPF7J/uMOvilyUOf2tiivFAyIDWe4L+CNVzuDEyPO2CGVnQp0pZQZOADMA3KBzcDVjQNbKRWlta7wrV8G3Ka1nt/eeSXQhRD13B4vR0urOXC8kgPHHf6wzyqqwt3KRdj6oB+aFMmQxAiGJhmt+aGJdqLD+3bXTWfvFJ0CHNJaZ/lOthxYCPgDvT7MfSKgA/e+CiGEj8VsYnBiJIMTI5k/pmF/ndtLdkmVEfQFRtgfOF7JkdJqjvqWT75ueq6EyBCGJEb6Q75+vd8ZMHFZRwI9FchptJ0LTG1eSCn1E+B/gBBgTmsnUkrdCtwKkJ6efqp1FUKcYUIsJn83C+Ma9te5vRwpqeJQodFdc6jQwaEiB98UVlHsqKPYUcqXh0ubnCs8xOwP94yECAYlRJARH8GghPA+c0G2I10u3wPma61v9m1fB0zVWt/eRvlrgIu11te3d17pchFCBJrXq8mvcBoB71u+KXLwTaGDkqq6Nt+XEBnCoHhfyPuWQb6w72kXZTvb5ZIHDGi0nebb15blwBMdr54QQgSGyaRIjQkjNSaMWcMTmxwrq6rzt+YPl1SRXVxFdnE12SX1rfo6Mo+UtThnclQog+Ij/K36+qBPj+t5Yd+R2mwGhimlMjCC/CrgmsYFlFLDtNYHfZvfBg4ihBA9SGxECJMi4pg0KK7J/vpWfXZxFYeLfUFfYqznlNZwvKKW4xW1LbpwwBhTPzAunPT4cAbGNQT9wPgIYsOt3d5nf9JA11q7lVK3A6sxhi0+p7Xeo5T6LZCptX4HuF0pdSHgAsqAdrtbhBCip2jcqp8xNKHJMY9Xc+xEjRH0JQ2Bf6S0mtzSGooqaymqrG21ZW8PtRhBHx9OelwEg+J9wR8fQb8oG6YueHyg3FgkhBCnwePV5JfXcLSkmiOl1RwpqeZoaZXxWlLd6nQI9WaPSOSFJVNO63PlARdCCBFgZpMiLTactNhwpjc7prWmtKrOGF5ZYoT9kdIqf/gPiA3vkjpJoAshRIAppYiPDCU+MpSz02NbHPe2crNUIMjclkII0c26ov8cJNCFEKLPkEAXQog+QgJdCCH6CAl0IYToIyTQhRCij5BAF0KIPkICXQgh+oig3fqvlCoCjpzm2xOA4gBWJ9Ckfp0j9eu8nl5Hqd/pG6i1TmztQNACvTOUUpltzWXQE0j9Okfq13k9vY5Sv64hXS5CCNFHSKALIUQf0VsD/elgV+AkpH6dI/XrvJ5eR6lfF+iVfehCCCFa6q0tdCGEEM1IoAshRB/RowNdKTVfKbVfKXVIKXVfK8dDlVKv+Y5/qZQa1I11G6CUWquU2quU2qOUuquVMrOVUuVKqe2+5dfdVT/f52crpXb5PrvF8/6U4THfz2+nUursbqzbiEY/l+1KqQql1E+blen2n59S6jmlVKFSanejfXFKqY+UUgd9ry2fWGCUu95X5qBSKuDP1W2jbo8opb72/fmtUErFtPHedn8XuriODyqlOy+AhwAABFNJREFU8hr9OS5o473t/n3vwvq91qhu2Uqp7W28t1t+hp2ite6RC8YDqb8BBgMhwA7grGZlbgOe9K1fBbzWjfXrB5ztW7cDB1qp32zgvSD+DLOBhHaOLwA+ABQwDfgyiH/WBRg3TAT15wecD5wN7G6078/Afb71+4A/tfK+OCDL9xrrW4/thrpdBFh8639qrW4d+V3o4jo+CPxvB34H2v373lX1a3b8/4BfB/Nn2JmlJ7fQpwCHtNZZWus6YDmwsFmZhcCLvvU3gLlKqa55FEgzWut8rfVW33olsA9I7Y7PDqCFwDJt2ATEKKX6BaEec4FvtNane+dwwGit1wOlzXY3/j17Ebi8lbdeDHyktS7VWpcBHwHzu7puWusPtdb1TyPeBKQF8jNPVRs/v47oyN/3Tmuvfr7suBJ4NdCf2116cqCnAjmNtnNpGZj+Mr5f6nIgvltq14ivq2ci8GUrh89VSu1QSn2glBrdrRUDDXyolNqilLq1leMd+Rl3h6to+y9RMH9+9ZK11vm+9QIguZUyPeFneSPG/7hac7Lfha52u69b6Lk2uqx6ws9vJnBca32wjePB/hmeVE8O9F5BKRUJvAn8VGtd0ezwVoxuhPHAP4C3u7l652mtzwa+BfxEKXV+N3/+San/3775u1YRBHH8M2BECBIUCxUtjNhbBBGxlKAiAcVCETRqk8LaJp1/gJ1YqCBIChEUXxEQ1FoURKOi6CsTwgtYKGLjj7HYOTju3T1O9HaPYz5wvLvdeezwfXNzt7P7RNYDM8C9ku7U+g2hYe7dur2+IjIP/AQWKkxSxsJ1YDewF1gllDXayGlGv523/n5qc0JfAXbmrndYW6mNiKwDJoDPUbwLY44RkvmCqt4v9qvqV1X9ZueLwJiIbInln6qu2Oca8IAwrc1TR+OmOQK8VNVBsSO1fjkGWSnKPtdKbJJpKSKzwDHgjD1whqgRC42hqgNV/aWqv4EbFWMnjUXLHyeAu1U2KTWsS5sT+gtgj4jssre4U0CvYNMDst0EJ4GnVQH9v7F62y3gvaperbDZmtX0RWQfQe8oDxwRGReRjdk5YfHsbcGsB5y13S77gS+50kIsKt+KUupXIB9n54CHJTaPgGkR2WQlhWlraxQROQxcBmZU9XuFTZ1YaNLH/LrM8Yqx69zvTXII+KCqy2WdqTWsTepV2VEHYRfGR8Lq97y1XSEEL8AGwlS9DzwHJiP6dpAw9V4CXtlxFJgD5szmEvCOsGL/DDgQ0b9JG/e1+ZDpl/dPgGum7xtgKvLvO05I0BO5tqT6ER4uq8APQh33ImFd5gnwCXgMbDbbKeBm7rsXLBb7wPlIvvUJtecsBrNdX9uBxVGxEFG/OxZfS4Qkva3oo10P3e8x/LP221nc5WyTaPgvh//133EcpyO0ueTiOI7j/AWe0B3HcTqCJ3THcZyO4AndcRynI3hCdxzH6Qie0B3HcTqCJ3THcZyO8AfL8s3bULWTjgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# Adagrad optimizer\n", "# Seems more well-behaved but slower. The validation loss is still decreasing after 20 epochs.\n", "model = models.Sequential()\n", "model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(16, activation='relu'))\n", "model.add(layers.Dense(1, activation='sigmoid'))\n", "model.compile(optimizer='adagrad', loss='binary_crossentropy', metrics=['accuracy'])\n", "history = model.fit(partial_x_train, partial_y_train,\n", " epochs=20, batch_size=512, verbose=0,\n", " validation_data=(x_val, y_val))\n", "pd.DataFrame(history.history).plot(lw=2);" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "25000/25000 [==============================] - 3s 114us/sample - loss: 0.3511 - accuracy: 0.8770\n", "Loss: 0.3511, Accuracy: 0.8770\n" ] } ], "source": [ "# Score is not better than RMSprop with early stopping, but could still improve with more epochs\n", "result = model.evaluate(x_test, y_test)\n", "print(\"Loss: {:.4f}, Accuracy: {:.4f}\".format(*result))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* Further tune the results by doing a grid search for the most interesting hyperparameters\n", " * Tune the learning rate between 0.001 and 1\n", " * Tune the number of epochs between 1 and 20\n", " * Use only 3-4 values for each" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "slideshow": { "slide_type": "slide" } }, "outputs": [ { "data": { "text/plain": [ "GridSearchCV(cv=3, error_score=nan,\n", " estimator=,\n", " iid='deprecated', n_jobs=None,\n", " param_grid={'epochs': [1, 10, 20],\n", " 'learning_rate': [0.001, 0.01, 1], 'verbose': [0]},\n", " pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n", " scoring=None, verbose=0)" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from tensorflow.keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\n", "from sklearn.model_selection import GridSearchCV\n", "\n", "def make_model(learning_rate=0.01):\n", " model = models.Sequential()\n", " model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", " model.add(layers.Dense(16, activation='relu'))\n", " model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", " model.compile(optimizer=optimizers.Adagrad(lr=learning_rate),\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])\n", " return model\n", "\n", "clf = KerasClassifier(make_model)\n", "param_grid = {'epochs': [1, 10, 20], # epochs is a fit parameter\n", " 'learning_rate': [0.001, 0.01, 1], # this is a make_model parameter\n", " 'verbose' : [0]}\n", "grid = GridSearchCV(clf, param_grid=param_grid, cv=3, return_train_score=True)\n", "grid.fit(x_train, y_train)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "Grid search results" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
mean_test_scoremean_train_score
param_epochsparam_learning_rate
11.00e-030.640.65
1.00e-020.860.88
1.00e+000.500.50
101.00e-030.860.88
1.00e-020.880.98
1.00e+000.500.50
201.00e-030.870.90
1.00e-020.871.00
1.00e+000.500.50
\n", "
" ], "text/plain": [ " mean_test_score mean_train_score\n", "param_epochs param_learning_rate \n", "1 1.00e-03 0.64 0.65\n", " 1.00e-02 0.86 0.88\n", " 1.00e+00 0.50 0.50\n", "10 1.00e-03 0.86 0.88\n", " 1.00e-02 0.88 0.98\n", " 1.00e+00 0.50 0.50\n", "20 1.00e-03 0.87 0.90\n", " 1.00e-02 0.87 1.00\n", " 1.00e+00 0.50 0.50" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "res = pd.DataFrame(grid.cv_results_)\n", "res.pivot_table(index=[\"param_epochs\", \"param_learning_rate\"],\n", " values=['mean_train_score', \"mean_test_score\"])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Exercise 2: Topic classification\n", "* Take the Reuters dataset from keras.datasets with 10000 words and the default train-test-split" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "hide_input": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "News wire 0: ? ? ? said as a result of its december acquisition of space co it expects earnings per share in 1987 of 1 15 to 1 30 dlrs per share up from 70 cts in 1986 the company said pretax net should rise to nine to 10 mln dlrs from six mln dlrs in 1986 and rental operation revenues to 19 to 22 mln dlrs from 12 5 mln dlrs it said cash flow per share this year should be 2 50 to three dlrs reuter 3\n", "News wire 5: ? the u s agriculture department estimated canada's 1986 87 wheat crop at 31 85 mln tonnes vs 31 85 mln tonnes last month it estimated 1985 86 output at 24 25 mln tonnes vs 24 25 mln last month canadian 1986 87 coarse grain production is projected at 27 62 mln tonnes vs 27 62 mln tonnes last month production in 1985 86 is estimated at 24 95 mln tonnes vs 24 95 mln last month canadian wheat exports in 1986 87 are forecast at 19 00 mln tonnes vs 18 00 mln tonnes last month exports in 1985 86 are estimated at 17 71 mln tonnes vs 17 72 mln last month reuter 3\n", "News wire 10: ? period ended december 31 shr profit 11 cts vs loss 24 cts net profit 224 271 vs loss 511 349 revs 7 258 688 vs 7 200 349 reuter 3\n" ] } ], "source": [ "from keras.datasets import reuters\n", "\n", "(train_data, train_labels), (test_data, test_labels) = reuters.load_data(num_words=10000)\n", "word_index = reuters.get_word_index()\n", "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", "\n", "for i in [0,5,10]:\n", " print(\"News wire {}:\".format(i),\n", " ' '.join([reverse_word_index.get(i - 3, '?') for i in train_data[i]]))\n", " # Note that our indices were offset by 3" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* We have to vectorize the data and the labels using one-hot-encoding" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "from keras.utils.np_utils import to_categorical\n", "x_train = vectorize_sequences(train_data)\n", "x_test = vectorize_sequences(test_data)\n", "one_hot_train_labels = to_categorical(train_labels)\n", "one_hot_test_labels = to_categorical(test_labels)" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Build a network with 2 dense layers of 64 nodes each\n", "* Make sensible choices about the activation functions, loss, ..." ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "model = models.Sequential()\n", "model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(64, activation='relu'))\n", "model.add(layers.Dense(46, activation='softmax'))\n", "model.compile(optimizer='rmsprop',\n", " loss='categorical_crossentropy',\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Take a validation set from the first 1000 points of the training set\n", "* Fit the model with 20 epochs and a batch size of 512\n", "* Plot the learning curves" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "x_val, partial_x_train = x_train[:1000], x_train[1000:]\n", "y_val, partial_y_train = one_hot_train_labels[:1000], one_hot_train_labels[1000:]" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "history = model.fit(partial_x_train,\n", " partial_y_train,\n", " epochs=20, verbose=0,\n", " batch_size=512,\n", " validation_data=(x_val, y_val))" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1fn48c+ZJTPZd5JAgICssokg4Aa4L8WiKIK7tOpXrdqq1fq1trWt3eze6q+K1h0FBLF+3aEgS8UlUHYIIAQIhOx7Mpnt/P64k8lCVphkMpPn/Xrd171z75k7z9xMnjlz7rnnKq01QgghQp8p2AEIIYQIDEnoQggRJiShCyFEmJCELoQQYUISuhBChAlLsF44JSVFZ2VlBevlhRAiJG3atKlYa53a2ragJfSsrCyys7OD9fJCCBGSlFKH2tomTS5CCBEmJKELIUSYkIQuhBBhImht6EKI3sXlcpGXl4fD4Qh2KAKw2+1kZmZitVo7/RxJ6EIIAPLy8oiNjSUrKwulVLDD6dO01pSUlJCXl8eQIUM6/TxpchFCAOBwOEhOTpZk3gsopUhOTu7yryVJ6EIIP0nmvcfJ/C1CLqG/uP4As5/ZwJqcwmCHIoQQvUrIJfT8Cgdb8yrYdawy2KEIIQIsJiYm2CGEtJBL6CPSjD/43oKqIEcihBC9S8gl9OFpsQDsLagOciRCiO6iteaRRx5h7NixjBs3jiVLlgCQn5/P9OnTOeOMMxg7dizr16/H4/Fw++23+8v++c9/DnL0wRNy3RaH9zNq6N8UVePxaswmOYkjRKBlPfZBt+w397ff6lS5d955hy1btrB161aKi4s566yzmD59Om+++SaXXXYZP/7xj/F4PNTW1rJlyxaOHj3Kjh07ACgvL++W2ENByNXQY+1W+sfbcbq9HCqpCXY4QohusGHDBm644QbMZjNpaWnMmDGDr7/+mrPOOouXX36ZJ598ku3btxMbG8vQoUM5cOAA999/Px9//DFxcXHBDj9oQq6GDkazy7EKB3sLqhmaKidRhAi0ztake9r06dNZt24dH3zwAbfffjsPPfQQt956K1u3buWTTz7hueeeY+nSpbz00kvBDjUoQq6GDo0nRvfJiVEhwtL555/PkiVL8Hg8FBUVsW7dOqZMmcKhQ4dIS0vjzjvv5I477mDz5s0UFxfj9Xq59tpreeqpp9i8eXOwww+akK2hA+wtlBOjQoSja665ho0bNzJhwgSUUjz99NOkp6fz6quv8vvf/x6r1UpMTAyvvfYaR48eZcGCBXi9XgB+85vfBDn64FFa66C88OTJk/XJ3uBiy5Fyrn72P4xKj+XjH0wPcGRC9E27d+9m9OjRwQ5DNNHa30QptUlrPbm18iHZ5NK0p4vL4w1yNEII0TuEZEKPtlnITIzE5dHS00UIIXxCMqEDjJALjIQQopkOE7pSaqBSao1SapdSaqdS6vutlJmplKpQSm3xTT/tnnAbDZchAIQQopnO9HJxAw9rrTcrpWKBTUqplVrrXS3Krddazwp8iK0b0c+ooe+TGroQQgCdqKFrrfO11pt9y1XAbmBAdwfWkcYmF6mhCyEEdLENXSmVBUwEvmxl89lKqa1KqY+UUmPaeP5dSqlspVR2UVFRl4Ntali/GJSCg8U1ON3S00UIITqd0JVSMcBy4Ada65aDkW8GBmutJwB/B95tbR9a64Va68la68mpqaknGzMAkRFmBiZG4fZqcqWnixCiC9xud7BD6BadSuhKKStGMl+ktX6n5XatdaXWutq3/CFgVUqlBDTSVsjY6EKEn6uvvppJkyYxZswYFi5cCMDHH3/MmWeeyYQJE7jooosAqK6uZsGCBYwbN47x48ezfPlyoPlNMpYtW8btt98OwO23387dd9/N1KlTefTRR/nqq684++yzmThxIueccw45OTkAeDwefvjDHzJ27FjGjx/P3//+d1avXs3VV1/t3+/KlSu55ppreuJwdEmHJ0WVcWO7fwK7tdZ/aqNMOlCgtdZKqSkYXxQlAY20FcPTYlm1u1C6LgoRaE/Gd9N+Kzos8tJLL5GUlERdXR1nnXUWs2fP5s4772TdunUMGTKE0tJSAH75y18SHx/P9u3bASgrK+tw33l5eXz++eeYzWYqKytZv349FouFVatW8fjjj7N8+XIWLlxIbm4uW7ZswWKxUFpaSmJiIvfeey9FRUWkpqby8ssv853vfOfUjkU36Ewvl3OBW4DtSqktvnWPA4MAtNbPAdcB9yil3EAdMF/3wJgC/hr6camhCxEu/va3v7FixQoAjhw5wsKFC5k+fTpDhgwBICkpCYBVq1axePFi//MSExM73PfcuXMxm80AVFRUcNttt7Fv3z6UUrhcLv9+7777biwWS7PXu+WWW3jjjTdYsGABGzdu5LXXXgvQOw6cDhO61noD0O5dJLTWzwDPBCqozvL3dCmUhC5EQHWiJt0dPvvsM1atWsXGjRuJiopi5syZnHHGGezZs6fT+zAaFQwOh6PZtujoaP/yT37yEy644AJWrFhBbm4uM2fObHe/CxYs4KqrrsJutzN37lx/wu9NQvZKUYDTUmMwKThUUku92xPscIQQp6iiooLExESioqLYs2cPX3zxBQ6Hg3Xr1nHw4EEAf5PLJZdcwrPPPut/bkOTS1paGrt378br9fpr+m291oABRg/sV155xb/+kksu4fnnn/efOG14vf79+9O/f3+eeuopFixYELg3HUAhndDtVjODk6PxeDUHiqSnixCh7vLLL8ftdjN69Ggee+wxpk2bRmpqKgsXLmTOnDlMmDCBefPmAfDEE09QVlbG2LFjmTBhAmvWrAHgt7/9LbNmzeKcc84hIyOjzdd69NFH+d///V8mTpzYrNfLHXfcwaBBgxg/fjwTJkzgzTff9G+76aabGDhwYK8dlTIkh89t6q7Xsvl0VwF/nX8Gs88I+vVOQoQsGT63Y/fddx8TJ07ku9/9bo+8Xp8YPrephnZ0GQJACNGdJk2axLZt27j55puDHUqbel+rfhfJIF1CiJ6wadOmYIfQofCpocvt6IQQfVzIJ/ShqdGYTYpDJTU4XNLTRQjRd4V8QrdZzAxOjsKrjVvSCSFEXxXyCR1gpJwYFUKI8Ejow30JPUdOjAoh+rCwSOgNY7rsk4QuRJ/RdFTFlnJzcxk7dmwPRtM7hElClxtGCyFEyPdDB8hKjsZiUhwpq6XO6SEywhzskIQIaeNeHdct+91+2/Y2tz322GMMHDiQ733vewA8+eSTWCwW1qxZQ1lZGS6Xi6eeeorZs2d36TUdDgf33HMP2dnZWCwW/vSnP3HBBRewc+dOFixYgNPpxOv1snz5cvr378/1119PXl4eHo+Hn/zkJ/6hBkJBWCT0CIuJISnR7CusZn9hNeMyu2ksZyFEt5k3bx4/+MEP/Al96dKlfPLJJzzwwAPExcVRXFzMtGnT+Pa3v91sRMWOPPvssyil2L59O3v27OHSSy9l7969PPfcc3z/+9/npptuwul04vF4+PDDD+nfvz8ffPABYAzgFUrCIqGD0eyyr7CavQVVktCFOEXt1aS7y8SJEyksLOTYsWMUFRWRmJhIeno6Dz74IOvWrcNkMnH06FEKCgpIT0/v9H43bNjA/fffD8CoUaMYPHgwe/fu5eyzz+ZXv/oVeXl5zJkzh+HDhzNu3DgefvhhfvSjHzFr1izOP//87nq73SIs2tChyRAAMja6ECFr7ty5LFu2jCVLljBv3jwWLVpEUVERmzZtYsuWLaSlpZ0wxvnJuvHGG3nvvfeIjIzkyiuvZPXq1YwYMYLNmzczbtw4nnjiCX7xi18E5LV6SljV0EH6ogsRyubNm8edd95JcXExa9euZenSpfTr1w+r1cqaNWs4dOhQl/d5/vnns2jRIi688EL27t3L4cOHGTlyJAcOHGDo0KE88MADHD58mG3btjFq1CiSkpK4+eabSUhI4MUXX+yGd9l9wiihyyBdQoS6MWPGUFVVxYABA8jIyOCmm27iqquuYty4cUyePJlRo0Z1eZ/33nsv99xzD+PGjcNisfDKK69gs9lYunQpr7/+OlarlfT0dB5//HG+/vprHnnkEUwmE1arlX/84x/d8C67T8iPh97A5fEy5qef4PR42fnzy4i2hc13lRA9QsZD73363HjoDaxmE0NTjfsF7peRF4UQfVBYVWOHp8Wy53gVOQVVTBiYEOxwhBDdbPv27dxyyy3N1tlsNr788ssgRRRcYZXQR/STIQCE6EvGjRvHli1bgh1GrxE2TS7QOEiXDAEghOiLwiqhyyBdQoi+LKwS+uDkaCIsJo5VOKhyuIIdjhBC9KiwSuhmk+K0VF8tXXq6CCH6mLBK6CDNLkL0Fe2Nh95XhWFClxOjQoie43a7gx2CX1h1W4SmCV1q6EKcrN2juueK0dF7dre5LZDjoVdXVzN79uxWn/faa6/xhz/8AaUU48eP5/XXX6egoIC7776bAwcOAPCPf/yD/v37M2vWLHbs2AHAH/7wB6qrq3nyySeZOXMmZ5xxBhs2bOCGG25gxIgRPPXUUzidTpKTk1m0aBFpaWlUV1dz//33k52djVKKn/3sZ1RUVLBt2zb+8pe/APDCCy+wa9cu/vznP5/SsYVOJHSl1EDgNSAN0MBCrfVfW5RRwF+BK4Fa4Hat9eZTju4kNDa5SA1diFASyPHQ7XY7K1asOOF5u3bt4qmnnuLzzz8nJSWF0tJSAB544AFmzJjBihUr8Hg8VFdXU1ZW1u5rOJ1OGoYvKSsr44svvkApxYsvvsjTTz/NH//4R375y18SHx/P9u3b/eWsViu/+tWv+P3vf4/VauXll1/m+eefP9XDB3Suhu4GHtZab1ZKxQKblFIrtda7mpS5Ahjum6YC//DNe9zAxCjsVhPHKx1U1LmIj7QGIwwhQlp7NenuEsjx0LXWPP744yc8b/Xq1cydO5eUlBQAkpKSAFi9ejWvvfYaAGazmfj4+A4TetM7GeXl5TFv3jzy8/NxOp0MGTIEgFWrVrF48WJ/ucTERAAuvPBC3n//fUaPHo3L5WLcuMDcIarDNnStdX5DbVtrXQXsBga0KDYbeE0bvgASlFIZAYmwi0wmxTC5YlSIkBSo8dADMY66xWLB6/X6H7d8fnR0tH/5/vvv57777mP79u08//zzHb7WHXfcwSuvvMLLL7/MggULuhRXe7p0UlQplQVMBFoOlDAAONLkcR4nJn2UUncppbKVUtlFRUVdi7QLRvSTE6NChKJ58+axePFili1bxty5c6moqDip8dDbet6FF17I22+/TUlJCYC/yeWiiy7yD5Xr8XioqKggLS2NwsJCSkpKqK+v5/3332/39QYMMFLeq6++6l9/ySWX8Oyzz/ofN9T6p06dypEjR3jzzTe54YYbOnt4OtTphK6UigGWAz/QWleezItprRdqrSdrrSenpqaezC46ZbicGBUiJLU2Hnp2djbjxo3jtdde6/R46G09b8yYMfz4xz9mxowZTJgwgYceegiAv/71r6xZs4Zx48YxadIkdu3ahdVq5ac//SlTpkzhkksuafe1n3zySebOncukSZP8zTkATzzxBGVlZYwdO5YJEyawZs0a/7brr7+ec889198MEwidGg9dKWUF3gc+0Vr/qZXtzwOfaa3f8j3OAWZqrfPb2megx0Nv6t+7C/juq9mcOyyZRXdM65bXECLcyHjoPWvWrFk8+OCDXHTRRW2WCfh46L4eLP8EdreWzH3eA25VhmlARXvJvLtJX3QhRG9VXl7OiBEjiIyMbDeZn4zO9HI5F7gF2K6Uahin8nFgEIDW+jngQ4wui/sxui0GrpX/JAxIiCTSaqaoqp7yWicJURHBDEcI0U1CcTz0hIQE9u7d2y377jCha603AO12+tRGu833AhXUqTKZFCPSYtiaV8HegmqmDEkKdkhChAStdYd9vHuTcB4P/WRuDxp2l/43kBOjQnSN3W6npKTkpBKJCCytNSUlJdjt9i49L+wu/W8gg3QJ0TWZmZnk5eXRnV2KRefZ7XYyMzO79JywTehy9yIhusZqtfqvcBShKWybXGSQLiFEXxO2Cb1/vJ0Ym4WSGicl1fXBDkcIIbpd2CZ0pRrHdJFmFyFEXxC2CR2anBgtlGYXIUT4C7mEnn08m4c+e4j86o4vRJV2dCFEXxJyCX1pzlJWHlrJ0r1LOywrPV2EEH1JyCX0G0ffCMDyvcup97R/snOkL6HvK6iSiyWEEGEv5BL6hNQJjE4aTVl9GR8f/LjdsmlxNmLtFspqXRRXO3soQiGECI6QS+hKKW4YZQwI/+aeN9uteSul/O3ocsWoECLchVxCB7hiyBUk2BLYVbKLrUVb2y3b0NNFTowKIcJdSCZ0u8XOnOFzAHhrz1vtlh3ecDu6QjkxKoQIbyGZ0AHmjZyHSZn49NCnFNcVt1nO33XxuNTQhRDhLWQTev+Y/szMnInb6+btvW+3Wa5pk4v0dBFChLOQTejQ2IXx7Zy3cXlcrZZJjbURH2ml0uGmsErGdBFChK+QTuhT0qdwWvxpFNUV8e/D/261jNHTRU6MCiHCX0gn9JZdGNsiV4wKIfqCkE7oAFeddhUx1hj+W/hfdpfsbrXMSOmLLoToA0I+oUdZo7h62NVA210Yh0uTixCiDwj5hA4wf9R8AD48+CHljvITtjdeLVotPV2EEGErLBL64LjBnDvgXOo99byz/50TtqfE2EiKjqCq3s3xSkcQIhRCiO4XFgkd4MZRRhfGJXuW4PF6Ttg+XO5eJIQIc2GT0M8bcB4DYwdyrOYYa/PWnrBdrhgVQoS7sEnoJmVi/kijLb21LozSF10IEe7CJqEDXD38aiItkXyZ/yUHyg802+bviy6DdAkhwlRYJfS4iDhmDZ0FnFhLb2hy2S9jugghwlRYJXTAf+Xoe9+8R5WzsXklKTqClBgbNU4PR8vrghWeEEJ0mw4TulLqJaVUoVJqRxvbZyqlKpRSW3zTTwMfZucNTxzOWelnUeeu471v3mu2raEdfZ/0dBFChKHO1NBfAS7voMx6rfUZvukXpx7WqWnowvjWnrfwaq9/vb+ni5wYFUKEoQ4TutZ6HVDaA7EEzMyBM0mPTudQ5SE2HtvoX984BIDU0IUQ4SdQbehnK6W2KqU+UkqNCdA+T5rFZGHeyHlA85Oj/iEACqWGLoQIP4FI6JuBwVrrCcDfgXfbKqiUukspla2Uyi4qKgrAS7dtzvA5RJgiWJ+3niOVRwAY0a9xTBevV3q6CCHCyykndK11pda62rf8IWBVSqW0UXah1nqy1npyamrqqb50u5LsSVw+5HI0msU5iwGIj7LSL9ZGnUt6ugghws8pJ3SlVLpSSvmWp/j2WXKq+w2EhlvUrdi/glpXLdDY7JIjQwAIIcJMZ7otvgVsBEYqpfKUUt9VSt2tlLrbV+Q6YIdSaivwN2C+7iVX7oxJHsP41PFUOav44OAHAJzePw6AD3fkBzM0IYQIuM70crlBa52htbZqrTO11v/UWj+ntX7Ot/0ZrfUYrfUErfU0rfXn3R925zV0YXxz95torbl56mAsJsWK/x6VWroQIqyE3ZWiLV06+FKS7cnsL99PdkE2g5KjuHHqILSG33+yJ9jhCSFEwIR9QrearcwdORdovEXd/RcOJyrCzKrdhWTnhlQXeyGEaFPYJ3SAuSPmYlEWVh9ezfGa46TG2rjjvCEA/O7jPTJYlxAiLPSJhN4vqh8XD74Yj/awNGcpAHdOH0pilJWvc8tYvacwyBEKIcSp6xMJHRq7MC7bu4x6Tz2xdivfu2AYAE9/nINHLjQSQoS4PpPQz0g9g1FJoyirL+OT3E8AuHnaYAYkRJJTUMW/thwNcoRCCHFq+kxCV0qd0IXRbjXz4CUjAPjjp3upd594c2khhAgVfSahA1wx5AribfHsLNnJhqMbALhm4gBGpMVwtLyORV8cDnKEQghx8vpUQrdb7P5RGB9Y/QBv7XkLk4JHLhsFwDNr9lPlcAUzRCGEOGl9KqED3DPhHm49/Vbc2s2vv/w1T/znCc4bHsfkwYmU1jh5cf3BYIcohBAnpc8ldIvJwiNnPcJvz/8tdrOd9755j9s/uZ07LkgE4MX1Byiurg9ylEII0XV9LqE3+NbQb/HGlW8wIGYAu0p28est/8NZo0qocXp4ZvX+YIcnhBBd1mcTOsDIpJEsmbWEc/ufS1l9GTnqj0QkrWfRl7kcLqkNdnhCCNElfTqhA8Tb4nn2ome5c9ydaLzY0j7AnPYWf1i5LdihCSFEl/T5hA5gNpl54MwH+PPMPxNpjsIav5VVFT9hzTe7gh2aEEJ0miT0Ji4efDFvzXqTGFMGZvtxHly/wN9fXQghejtJ6C2clnAab816C11zOh5Vy72r7uWFbS/IiIxCiF5PEnorshKT+e7wn1NfdDFaw9/++zce/OxBalw1wQ5NCCHaJAm9DXdOH0Zs3ZXU5t2G3RzNvw//mxs/uJGDFXLhkRCid5KE3oYYm4X7LxyGp3oUMcUPc1r8aRyoOMCNH9zImsNrgh2eEEKcQBJ6O26YOojMxEgOHo/iuoynuWTwJVS7qnlgzQPcu+pe1uetx6u9wQ5TCCEASejtslnMPHypMbzu/1t9hF+d8zQPTXqICFME64+u595/38usFbN4deerVNRXBDlaIUR3q6ivYPXh1Wwp3ILb6w52OCdQweq9MXnyZJ2dnR2U1+4Kr1dz5d/Ws+d4FU98azR3nD+UMkcZ7+x7h6U5SzlWcwwAu9nOlUOvZP7I+YxOHh3kqIUQgaC15mDFQT7L+4y1R9aypWiL/1d5jDWGs9LPYlrGNKb1n8aQuCEopbo9JqXUJq315Fa3SULv2Jo9hSx45WsSoqyse/QC4uxWADxeD+vy1rE4ZzGfH/vcX35C6gTmj5rPpYMvJcIcEaywhQhLde46rCYrFpOlW/bv9DjJLshmXd461h5ZS151nn+bRVmY0G8CJXUl5FbmNnteWlSaP7lPy5hGSmRKt8QnCf0Uaa2Z9/wXfJVbyn0XDOOHl408oUxuRS5Lcpbwr/3/ospVBUCSPYlrh1/L9SOvJz06vafDFiJsuL1uNhzdwPJ9y1mftx6TMjEodhBD4oeQFZ9FVlyWfzkuIq7L+y+uK2Z93nrW5a3j82OfU+tuHMsp0ZbI+ZnnMyNzBmf3P5vYiFgA8qvz+SL/Czbmb+TL/C8pdZQ22+fwxOFMy5jG2RlnMyltElHWqFM7CD6S0ANg06Eyrv3H50Razax9dCb9Yu2tlqt11fLBwQ9YvGcxe8v2AmBSJi4YeAHzR81navrUHvlZJkQ4OFJ5hBX7V/Du/ncpqisCjP+n9jojJNmTjOTuS/INy/1j+vtr9Vpr9pTuYW3eWtblrWN78fZm+xiZOJLpmdOZMXAGY5PHYjaZ243Tq73sK9vnT/CbCzZT567zb7eYLExInWAk+P5nMyZ5zEn/wpCEHiB3vpbNyl0F3DJtML+8emy7ZbXWbC7czOI9i1l1aBVubZxAGRo/lHkj5zF72GyirdE9EbYQIaXeU8+qQ6t4Z987fHX8K//6rLgs5gyfw1WnXUWUJYrcylxyK3LJrczlYMVB/2OHx9Hqfq0mK4NiBzEgdgB7SvZQWFfo32Yz25iaMZUZmTM4f8D5ZMRknNJ7cHqcbC3aysZjRu19R8mOZl9Cg2IH8f41759U5U4SeoDsK6jisr+sw6QUnz44naGpMZ16XlFtEcv2LWNZzjL/hyjGGsPVw67mxlE3MjBuYHeGLURIyCnNYfm+5Xxw4AMqnZWA0dng0qxLuXb4tUzsN7HDBOjVXgpqCjhYedBI8hW5HKw05gW1Bc3K9ovqx4zMGczInMGUjClEWiK77b1VOiv5+vjX/gQ/Omk0T894+qT2JQk9gB55eytvb8pjaGo0y+4+h6Tozp/0dHldrDm8hkW7F7G5cDMACsWMzBncdPpN0hwj+pwqZxUfHfyId/a9w86Snf71Y5LHMGf4HK4YcoW/zfpU1bpqya3M5UjVEQbFDmJU0qig/b85Pc6T7jAhCT2AKupczHt+I3uOVzEhM55Fd04jxtb1trDdJbtZtHsRHx78EJfXuDH1sIRh3DT6JmYNnYXd0nobvRChrqE58p197/Bp7qf+JpLYiFhmDZ3FnOFzGJU0KshR9l6nlNCVUi8Bs4BCrfUJDcfK+Ir7K3AlUAvcrrXe3FFQoZrQAQorHVz73OccKa3jvGEp/PP2ydgs7Z80aUtJXQlv732bJTlLKK4rBoybblw3/Drmj5ovvWNESHJ6nBTUFHC89jj5NfnkV+eTX5PP8ZrjHKw46L9+A2BK+hTmDJ/DRYMukopMJ5xqQp8OVAOvtZHQrwTux0joU4G/aq2ndhRUKCd0gNziGq57biPF1fV8a3wGf5s/EbPp5H++uTwuPjn0CYt2LWJHyQ4AzMrMxYMv5ubRNzMhdYI0x4heQWtNqaOU47XHOV7tS9g1jQk7vybfXzlpS2pkKlcPu5prhl0j55C66JSbXJRSWcD7bST054HPtNZv+R7nADO11vnt7TPUEzrAjqMV3LDwC6rq3dw8bRC/nD32lJOu1pqtRVtZtHsRKw+txKM9gNGmeNPom7g863KsZmsgwhd9mMfrodpVTaWzkkpnJVXOKirrffMW6ypdlVTVN66vdFZ2eNm7WZnpF9WPjOgM0qLTyIjO8E/p0emclnBat10YFO66O6G/D/xWa73B9/jfwI+01idka6XUXcBdAIMGDZp06NChLryN3umLAyXc+tJXON1evn/RcB68ZETA9n285jhLcpawbO8yyuvLAUiJTOH6Eddz7oBzGZk0EpvZFrDXE+HF6XFyuPKwv5dHQ7e+Q1WHTnnsobiIuGYJOj063XgcY6xLiUyRhN1Nek1CbyocaugNPtl5nHve2IRXw8+/PYbbzskK6P4dbgcfHPiAN3a/wf7y/f71FpOFEYkjGJs8lrEpxjQ0fmiHF0GI8KG1priuuFlf7IbuesdqjrV7AU6sNZbYiFjibHHERcQZyy3nvm0NU2yE8Rxp6w4eaXLpAUu/PsKjy7ehFPxl3hnMPmNAwF9Da81Xx7/i/QPvs7gxRX8AAB5KSURBVL1oOwcqDqBp/veLtERyevLpjEsZ50/y/aP7S/v7qdAatLdx8nrA6wKP2zd3Nc79y+5W1jXd5gbtMfalPeD1Nj72bdMeD9VeB2WuWko9tZS46yj11lHsdnDYU02uu4ZcTzXVuvXmDxMwwBxNliWGIZZYsiyxZFljybLEkIQVM9p4rYbX9Lp978/dLA5juUUZ7TWOC9o3p8lyR3NAKUCBMhnLynTiY+UbDLbVMqYWZZs+p2WZhonGZe1t9Zgbfw9vk79LK38fXzOo8Tq+99HsPbU2b7E9ZSRcGfh+6IH4TfQecJ9SajHGSdGKjpJ5OLr+rIGU1Dj53cd7eHjpVhKiIpgxIjWgr6GUYmrGVKZmGOeca1w17CrZxY7iHWwv3s7O4p0cqznGpoJNbCrY5H9ekj2JMcljGJcyjjEpYzg9+XSS7EmYVAiNnqw1uOvBVWtMzhpwVkN9deNyZx47a4ypaWLyepokbU+L9V7g5Lr2egE34FYKlwI3yr9cYTJTYjZRajZTZjZRajJTajZRYjZTajaWS81mXJ34Io7zeMhyuclyuRjicjPE5SLL5WKgy40MDddL1Vd3y24708vlLWAmkAIUAD8DrABa6+d83RafAS7H6La4oKPmFgi/GjoYNehff7ibF9YfJNJq5s07pzJxUGKPxlBSV8LOkp1sL97OjuId7Cje4W9/b0qhiLZGExMRQ4w1htiIWP+8YTkmIoZYa6wxb7IuLiKOJHtS+xdGeD3gqIC6MqgrN+aO8sbHrlpw1bUyb2u5Fo2m0mSixGzCosGuNTbtxa41EdpfTzopGqhTigqTiTKziXKzmXLfcoXJ3GRupsJswqlMuJXCrcDlS9Ru5UvgvskbgB9FkcpCktlOsslOkiXSmMxRZFrjyIqIZ4g1gUSTzfcLzPe/3LTG3NZjkxlMFlC+ucnkm7eyzv/YbEzK3Fgr7nQtlRZlG75AafzSbKj1N3vccl2LcieUbdjWcl2Tsl6PEX/De2l4b83WNcxNLR775g3Hs8u/TnxzezxktlrJ7pBcWNSDvF7NI8u2sXxzHglRVt7+n7MZnhaYK91Ohtaao9VH/cl9e/F29pXt848IeSriTRGkKCsp2kSKV5PqdpPiqifFUUOqo4oUj4cUj4dYr2432Wqg0mSi0GymyGymyOKbt7LsbKPGqgA7JmzKjN1kNSZzBDaTDbvFht0Sic0Sid0ahdUSSbW7jnJnFeXOCsqclVQ4K6j3OE/5mLRkURYspuaT1WQl3hZPkj3phCk5MpkkexKJ9kQSbYkBG6FPhA9J6D3M5fFy9+ub+PeeQjLi7Sy75xwGJHTfOBEno6HbWnV1IdUVuVRVHKG6+hhVNcepri2huq6UKmelUcbtoEppqkwmqk2KKpPRHODuZLu8TUOKspBispFiiSLBEk0lXgq99RR76ihy1+D0t0u2L8YaQ5I9Ca/24vA4qHfX4/A4/FfbnooIUwQJ9gQSbYkk2BJIsCcYc1sCifZE4m3xJNqMeYQ5wkjOynpCwvZPyiLnLkTASUIPAofLwy3//JKvc8tOatyXgHE5oCwXSr+Bkm+MeelBqMqHquNQX9m5/djiIDYDYtMhNh1vZBLltiiKzBZKzCaKlJci7aLE66DIVUORs4ISRwlFtUXNxpZuS4w1htSoVPpF9iMlKsWYR6bQL6r5vK0aq8frod5TT527jnqPkeQdbkfjOne9f9nldRFjjTkhaUdaIiUBi15PEnqQBGrclw65nScm7ZJvoPQAVOTR7kk9sw3iMnzJOqNJ0s5oXB+TBrbOjSzZmlpXLcV1xRTVFVFcV0y5o5x4WzypUamkRqa2m6iFEM1JQg+iQI77Ql05HN8OBTuhZH9j4q444jvh0wplhoRBkHwaJJ3mmw+FuAFG4o5MbOxWJYTo9bq726JoR784O69/ZyrXPbeRDfuLeWjJVv52QyfGfakuhPytjdPxbUYtvDXKBAmDWyRt3zxhEMhQAUL0CZLQe0BWSjSvLDiLGxZ+wQfb80mMtjaO+6I1lB9uTNr5WyF/G1QfP3FHZhukjYH0cZAyojFxJw4GiwwBIERfJwm9h4wdEM8Lt03msZc+oPKr/7A+r4TzYo5hOr7N6J/dki3OSNzp4yFjAmSMN5K41La7jdYaXVeHt7YWb02NMW+Yamrw1viW6+qMLtUWK8piQVktKIsFLJa211ktKLPZeGy1YrLbMUVGoiKjMEVFokwhdJGX6LUkoXcnrY027kP/gUOfM+3Qf/jMcsTYVuybAKJSGpN2xgQjiScOMS5qOAVepxNvZSWeyiq8VU3m1dVGAjFbUGYTmM0oswXMJpTZgrIYF1Aoi9lIQv5yvm3KhHa7wOVCt5yczk6scxv7sVoxRUSgrFaU1Qq+ecO6hseqyXqsVpTFauyz3oG3vh7tm7yOenS9o9myt74e7ahHO33rHA689Q50bR2e2hp0TW2zxE2QzikpX4I3RUaioiIxRUa1/jgqEhUZacTp8aI9HvC40R4v2uMGtzFkAF4P2u0x1rUoh8fj/2Jpd2ryt2lcZ3xhaY/X97ds5e/dyt+cFn9/U1QU5sREY0pIwJyYgDkhAUuTdabIwHb11R6P7+9fb8TkMY6Vdrt9y17fMfI02dbi+DasO+H4ttiH24P2Nu6n5fOs/TNIuvXWgL4/kIQeWF4vFO3xJXAjiVPd/D6G2BOo7DeZJUdT2FiXSUHUCH457xLOHJzU4e49VVXU5+RQ/80BPBUVeKuq8FRV4q2swlNVZSTvJut0fX03vdHwpex2TFFRmKKjjXnL5SijRq21Brcb7XKj3Q2Tq/V1zR4b63SdA29dHd66OnRdHdrhwONw4CkrC/Yh6DWU3e5P7hZfwjcnGI+19hrH0FGHdtTjdTiMX1cOh5G0Hb5tdb4EXleHdgb+wrGTZZ8wXhJ6r+P1GL1OGpL3oc+hrrR5magUyDoXBvumfqcTZzJxTXU9KxdtZufBUuYv/JJfzB7D/CmDANBeL67Dh3HsyaF+b44xz8nBdfRo1+KzWDDHxWGOjcXUZG6K9nURPKE217RG0aLW5/UaycpXHouvttasBhfRWItrc10EymxGez2drM0btTtvk/W43Ma+7HaULQKTzW4kYlsEymZH2WyY7Dbfsm+7f50xmaKiG5N1dGOyVuaeH6lSe71GEvIleG9trZGc6urw1tbhrfM9rvWtq6s1fmH5fkVhNqP8yx3/6sJkMv7O7R17Zzu/uFwuMJsafzF1WKtvsc5iwVtTg6esHE95Ge6yMjzl5cbjhuXSUrTDgTs/H3d+PgGpmiiFiozEZLMZvwAtvuPW0BRmMhlzc8Mv04ZtRjksxjH1/1o1m088vk2e1+yYN9unBUu/foF4Rye+Rem22EVFOZDzkZHED39x4oU5sf19CfwcGHwepAxvs1ugy+Plt8uy+WLVV2RVHOMyexVjHYU49+1D19WdUF5FRGAbPhzb8OFYUpIxxcZhio3BHBuHOS4WU4u5stvlQhkRkrTW6NpaPOXluJsmet8cswmTPRJlt2GyR2KKtKP8c1/zld23bLejIiONL5Yw+H+QbounQmso3A27/gW73jWaVJpKGAxZ5/kS+LmQmOVP4NrjMT6ApaW4S0t9NZBS3IWFOHL2Ur9nD9cdPcp1TXbXUBOxpKVhGzUS+4iRxnzUKCIGDzZOtAkR5pRSqOhoTNHRWAcEfijqcCXZoTVaQ+Eu2PmukcSL9/o3ec0JOFNm4IkdjTtiAB6Hwn2oFM9/9+Mp+9pI3r4k7qmoMNrV26EiIrANG0Z1ZhZLS+1stfWjNjOLP94xg+EDE7r7nQohwogk9AZaG+3hu/6F3r4C5+FcnFUWY6rth9OdgrNc4y6tAL72TR0zxccbZ+6TkjAnJWJJTMKckozttGHYR44gYsgQf607rcrBPW9sZv+hMuY+v5HfXDOOaydldt97FkKElT7bhq69XtzHj+PcvBrn1x/j3L2F+qIanFUWXDVm0G0M0xoRgTUzE0tKCuakJCxJiZgTfck6Kan5ckJCl5tInG4vP/+/nSz68jAAC87N4vErR2M1Sz9lIYS0oQNGe3bd1q1UrVxJzbo1OA/noV0th2z13SdRKawD+hORldU4DRlCRFYW1oz0bu0JEWEx8atrxjGmfzw/e28HL/8nlz35VTxz40SSY+RqUCFE28I6oWunk5ovv6Rq5SqqVq/GU1zcbLvZ5iEiwUTE4IFEjDmLiIkzsA0ZinXQIEy24CbPG6cOYmR6DHe/sZmNB0r49jP/4flbJjF2QHxQ4xJC9F5h1+Tiramhev16qlauonrtWrzVjffus8ZbiE0vJ2YQ2GfMwTz5ehg0zbitVC9VUOngf17fxJYj5ditJn537fhuuQG1ECI0hH2Ti7usjOrVq6lauYqazz9vdkWYbcQIYqdPI7b2X9g8u1ExqXDjUhhwZhAj7ry0ODtL/mcaP3l3B0uz8/j+4i3sOFrBjy4fhUXa1YUQTYRsQncdO0bVqlVUrVxF7aZNjd0DlSJy4kRiL76Y2IsvIiKyDhZdB94jxuiENy+HpCHBDb6LbBYzv7t2POMGxPPz/9vFC+sPsvFACb+5ZjzjMqUJRghhCLkml7LFSyh/+20cO3c2rrRaiZ461UjiF12IJTXVWH/oc3hrvnH3+cwpcMNiiE4O0DsIjq8OlvLgki0cLa/DpOD2c4bw8KUjiO6OOyEJIXqdsGpyqf/mGxw7d6Kioog5/3xiL76YmBnTMcfFNS+481145y7w1MOoWTDnBYgI/ducTRmSxMqHpvPnlXv554aDvPSfg3y8I59fzB7LxaenBTs8IUQQhVwNvX7/fpyHjxB9ztmY7PbWC238f/DJ44CGs+6AK57u1Sc+T9aOoxX87zvb2X60AoDLx6Tz5LfHkB7fxnERQoS8vnNPUa8XVv4ENj5jPL74STj3B2F9z0yPV/Pq57n84dMcap0eYmwWHr18JDdNHdzxbe6EECGnvYQePt0k3PWw/LtGMjdZ4ZqFcN6DYZ3MAcwmxXfOG8Kqh2Zw8eg0quvd/PRfO7n2H5+zO7+y4x0IIcJGeCT0ujJ4fQ7sfAciYuHmZTBhXrCj6lH9EyJ54dZJPHfzmaTF2dhypJyr/r6B3360hzpnyytihRDhKPQTekUevHQFHNoAsRnwnY9g6MxgRxUUSikuH5vByodmcOvZg/FozXNrv+Gyv6xj3d6iYIcnhOhmoZ3QC3bCi5dA0W5IHQXfXWncWLmPi7Nb+cXssSy/5xxGpcdyuLSWW1/6iu8v/i/F1XJbOiHCVegm9ANr4aXLoeqYcWOJ73wMCQODHVWvcuagRP7v/vP40eWjsFlM/GvLMS7641qWfH0Yrzc4J8OFEN2nUwldKXW5UipHKbVfKfVYK9tvV0oVKaW2+KY7Ah9qE9vehjeuNW7/NuYauPkdiEzs1pcMVVaziXtmnsanD07n/OEpVNS5+NHy7Vz1zAZW7ykgWL2chBCB12G3RaWUGdgLXALkYdzZ4Qat9a4mZW4HJmut7+vsC590t8UNf4FVPzOWp30PLn3KuOmt6JDWmve2HuPXH+6moNJoejlzUAI/vHQk5wxLCXJ0QojOONVui1OA/VrrA1prJ7AYmB3IALukrhRQcNmv4fJfSzLvAqUUs88YwNpHLuCJb40mOTqCzYfLufHFL7lh4RdsOlQa7BCFEKegMzX064DLtdZ3+B7fAkxtWhv31dB/AxRh1OYf1FofaWVfdwF3AQwaNGjSoUOHuh6x1wt5X8OgqV1/rmimpt7NK5/n8vzab6h0uAG4YGQqD186UsZdF6KX6okLi/4PyNJajwdWAq+2VkhrvVBrPVlrPTm1YQCtrjKZJJkHSLTNwvcuGMb6H13I/RcOIzrCzJqcImb9fQP3vLGJvQVVwQ5RCNEFnUnoR4Gm3Ucyfev8tNYlWuuG/nAvApMCE57oCfGRVh6+dCTrHr2AO88fgs1i4qMdx7nsL+v4weL/kltcE+wQhRCd0JmE/jUwXCk1RCkVAcwH3mtaQCmV0eTht4HdgQtR9JTkGBs//tbprHv0Am6ZNhiLSfHulmNc9Ke1PLZ8G0fL64IdohCiHZ0anEspdSXwF8AMvKS1/pVS6hdAttb6PaXUbzASuRsoBe7RWu9pb5/ddQs6EThHSmv5++p9LNuUh1dDhNnEjVMHce8Fp9EvVkZ0FCIY+s5oi6JbHCiq5i+r9vF/246hNditJq4a359rJ2UyJSsJk4zqKESPkYQuAmLP8Ur+9OlePt1V4F+XmRjJnDMzufbMAQxOjg5idEL0DZLQRUB9U1TNO5vzWLH5KMcqHP71Z2Ulcu2ZmVw5PoM4uzWIEQoRviShi27h9Wo2Hihh+aY8PtpxnDqXMUyvzWLisjHpXDspk/OGpciNNoQIIEnoottV17v5aHs+yzfn8cWBxitO0+JsXD1xANedmcnwtNggRihEeJCELnrUkdJaVvz3KMs353GopNa/fkJmPNdOyuSq8f1JjI4IYoRChC5J6CIotNZsOlTGsk15fLAtn6p6Y3gBq1kxZUgSM0f0Y+bIVIb1i0GF+a0ChQgUSegi6BwuD5/sPM7yzUfZsK+IpsOxD0iIZMbIVGaOSOWcYSnE2CzBC1SIXk4SuuhVSmucrN9XxGc5RazbW0RJjdO/zWpWTB6cxMyRqcwc2Y8RaVJ7F6IpSeii1/J6NTuOVfBZThGf5RSy5Uh5s9p7RrydGSNSmTkylXOHpRAr3SFFHycJXYSM8lon6/YVszaniLV7i5rdA9ViUkwanMjMkf2YNjSJMf3jibDIePiib5GELkKS16vZlV/JZzmFfJZTxObDZc1q7zaLiTMGJjA5K5HJWUmcOSiR+EipwYvwJgldhIWKWhcb9hezfl8RX+eW8k1R82F9lYIR/WJ9CT6RyYOTyEyMlDZ4EVYkoYuwVFrjZNOhMrIPlZKdW8b2vAqcHm+zMmlxNiZnJTF5cCJnZSUxKj0Wi1maaUTokoQu+gSHy8P2oxVk55aRnVvKpsNllNe6mpWJjjBzxqAERqXHMaxfjDGlxsiFTiJkSEIXfZLXq/mmqJrsQ2V8nVvKpkNlza5cbSo5OoLTUmM4rSHJ+6aMOLsMDyx6FUnoQvgUVjn47+Fy9hdWs7+wmm+KqvmmsJoap6fV8lERZoamRjMs1Ujwp/nmAxIjiYqQC6BEz5OELkQ7tNbkVzj8Cb4x2dc06zbZUnyklYx4OxnxdtLjI/3LGfGRpPuWo+WqVxFg7SV0+bSJPk8pRf+ESPonRDJ9RGqzbeW1Tn+S/6aohv2F1RwoquZYhYOKOhcVdS72HK9qc99xdkuzBJ/hS/ypcTb6xdpIjbWRHG2TIYZFQEhCF6IdCVERTBqcxKTBSc3Wa60prXGSX+Egv8LB8Yo6/3J+k+VKh5tKRxU5BW0nfbNJkRwdQWqskeT7xdqN5SZJv2Gd3Wru7rcsQpgkdCFOglKK5BgbyTE2xg6Ib7WM1pqyWhfHyus4XuEgv9JBvm+5sKqeoqp6CqsclNW6KKyqp7Cqnp0dvG6s3UK/WBsJURHE2S3ER1qJi7QSZ7f6li3E2Y118b71cZEWYu1W+RXQB0hCF6KbKKVIio4gKTqizaQP4HR7Ka42EnphpYOi6noKK+sb51UOiqqMx1UON1UON1DT5v7aEmuzGMk/0ur/Mmg6JUQ1fhE0neIirVil735IkIQuRJBFWEz+Nvz2eL2a8joXRVX1VNS5qKxzUekw5hV1bv+yMXcbZXzrqurd/uloeV2XY4yOMPuTe3ykFZvVjMWkMJsUFpPCYja1eKywmEzNHptNjWVsFhMRFhMRZhM2q4kIs5kIi6lxvW/ZZml9m8Wk5ArgVkhCFyJEmEyNNf6u8no1VfVuX/Jv+kXgorzW5T/B2zBVtnhc4/RQ4/Q0uyl4MCkFVpMJq1lhtZiwmExE+JcVVnNj4m+5bPV9OditZuwWM3arb9k/902Wpo+bl1dKodFoDVqDV2s0xnEG3+Mm641yvjJaY7eYyUqJDvhxkYQuRB9gMil/E8rALj5Xa011vbtZgnd5NG6PF7dX4/FqXB4vHq/2P26+TePxGo/dHo3L68Xl1jg9HpxuL/VuL07f1LBc7/FS7/Lg9LSyze3BqzG2eYA2riHozc4YmMC73zs34PuVhC6EaJdSili7lVi7lczEYEdjaPgScXm8xpeEx4uzE8vGY029y4PDbXxpOFwe6lweHC4vjoa52+Pb5lvnNpbrnB7q3R60No6LUmBSoFDG3LfOWK9QNFmHb52CQUlR3XJcJKELIUKO2aQwm8zSjbMFOXUthBBhQhK6EEKECUnoQggRJiShCyFEmOhUQldKXa6UylFK7VdKPdbKdptSaolv+5dKqaxAByqEEKJ9HSZ0pZQZeBa4AjgduEEpdXqLYt8FyrTWw4A/A78LdKBCCCHa15ka+hRgv9b6gNbaCSwGZrcoMxt41be8DLhIyXW5QgjRozqT0AcAR5o8zvOta7WM1toNVADJLXeklLpLKZWtlMouKio6uYiFEEK0qkcvLNJaLwQWAiilipRSh05yVylAccACC7zeHh/0/hglvlMj8Z2a3hzf4LY2dCahH4Vmwz9k+ta1ViZPKWUB4oGS9naqtU5tb3t7lFLZbd2CqTfo7fFB749R4js1Et+p6e3xtaUzTS5fA8OVUkOUUhHAfOC9FmXeA27zLV8HrNbBulmpEEL0UR3W0LXWbqXUfcAngBl4SWu9Uyn1CyBba/0e8E/gdaXUfqAUI+kLIYToQZ1qQ9dafwh82GLdT5ssO4C5gQ2tXQt78LVORm+PD3p/jBLfqZH4Tk1vj69VSlpGhBAiPMil/0IIESYkoQshRJjo1Qm9N48ho5QaqJRao5TapZTaqZT6fitlZiqlKpRSW3zTT1vbVzfGmKuU2u577exWtiul1N98x2+bUurMHoxtZJPjskUpVamU+kGLMj1+/JRSLymlCpVSO5qsS1JKrVRK7fPNW71vj1LqNl+ZfUqp21or003x/V4ptcf3N1yhlEpo47ntfh66Mb4nlVJHm/wdr2zjue3+v3djfEuaxJarlNrSxnO7/fidMq11r5wwetR8AwwFIoCtwOktytwLPOdbng8s6cH4MoAzfcuxwN5W4psJvB/EY5gLpLSz/UrgI4y7Y00Dvgzi3/o4MDjYxw+YDpwJ7Giy7mngMd/yY8DvWnleEnDAN0/0LSf2UHyXAhbf8u9ai68zn4dujO9J4Ied+Ay0+//eXfG12P5H4KfBOn6nOvXmGnqvHkNGa52vtd7sW64CdnPikAi93WzgNW34AkhQSmUEIY6LgG+01id75XDAaK3XYXS9barp5+xV4OpWnnoZsFJrXaq1LgNWApf3RHxa60+1MeQGwBcYF/8FRRvHrzM68/9+ytqLz5c7rgfeCvTr9pTenNADNoZMd/M19UwEvmxl89lKqa1KqY+UUmN6NDDQwKdKqU1Kqbta2d6ZY9wT5tP2P1Ewj1+DNK11vm/5OJDWSpneciy/g/GrqzUdfR66032+JqGX2miy6g3H73ygQGu9r43twTx+ndKbE3pIUErFAMuBH2itK1ts3ozRjDAB+Dvwbg+Hd57W+kyMoY+/p5Sa3sOv3yHf1cffBt5uZXOwj98JtPHbu1f29VVK/RhwA4vaKBKsz8M/gNOAM4B8jGaN3ugG2q+d9/r/p96c0Lsyhgyqk2PIBJJSyoqRzBdprd9puV1rXam1rvYtfwhYlVIpPRWf1vqob14IrMD4WdtUZ45xd7sC2Ky1Lmi5IdjHr4mChqYo37ywlTJBPZZKqduBWcBNvi+dE3Ti89AttNYFWmuP1toLvNDG6wb7+FmAOcCStsoE6/h1RW9O6L16DBlfe9s/gd1a6z+1USa9oU1fKTUF43j3yBeOUipaKRXbsIxx4mxHi2LvAbf6ertMAyqaNC30lDZrRcE8fi00/ZzdBvyrlTKfAJcqpRJ9TQqX+tZ1O6XU5cCjwLe11rVtlOnM56G74mt6XuaaNl63M//v3eliYI/WOq+1jcE8fl0S7LOy7U0YvTD2Ypz9/rFv3S8wPrgAdoyf6vuBr4ChPRjbeRg/vbcBW3zTlcDdwN2+MvcBOzHO2H8BnNOD8Q31ve5WXwwNx69pfArjblTfANuByT38943GSNDxTdYF9fhhfLnkAy6MdtzvYpyX+TewD1gFJPnKTgZebPLc7/g+i/uBBT0Y336M9ueGz2FDz6/+wIftfR56KL7XfZ+vbRhJOqNlfL7HJ/y/90R8vvWvNHzumpTt8eN3qpNc+i+EEGGiNze5CCGE6AJJ6EIIESYkoQshRJiQhC6EEGFCEroQQoQJSehCCBEmJKELIUSY+P9QFrV3nNnTpQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "pd.DataFrame(history.history).plot(lw=2);" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Create an information bottleneck: rebuild the model, but now use only 4 hidden units in the second layer. Evaluate the model. Does it still perform well?" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2246/2246 [==============================] - 0s 88us/sample - loss: 2.0950 - accuracy: 0.6901\n", "Loss: 2.0950, Accuracy: 0.6901\n" ] } ], "source": [ "model = models.Sequential()\n", "model.add(layers.Dense(64, activation='relu', input_shape=(10000,)))\n", "model.add(layers.Dense(4, activation='relu'))\n", "model.add(layers.Dense(46, activation='softmax'))\n", "\n", "model.compile(optimizer='rmsprop',\n", " loss='categorical_crossentropy',\n", " metrics=['accuracy'])\n", "model.fit(partial_x_train,\n", " partial_y_train,\n", " epochs=20,\n", " batch_size=128, verbose=0,\n", " validation_data=(x_val, y_val))\n", "result = model.evaluate(x_test, one_hot_test_labels)\n", "print(\"Loss: {:.4f}, Accuracy: {:.4f}\".format(*result))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Exercise 3: Regularization\n", "* Go back to the IMDB dataset\n", "* Retrain with only 4 units per layer\n", "* Plot the results. What do you observe?" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "from keras.datasets import imdb\n", "import numpy as np\n", "\n", "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)\n", "\n", "def vectorize_sequences(sequences, dimension=10000):\n", " # Create an all-zero matrix of shape (len(sequences), dimension)\n", " results = np.zeros((len(sequences), dimension))\n", " for i, sequence in enumerate(sequences):\n", " results[i, sequence] = 1. # set specific indices of results[i] to 1s\n", " return results\n", "\n", "# Our vectorized training data\n", "x_train = vectorize_sequences(train_data)\n", "# Our vectorized test data\n", "x_test = vectorize_sequences(test_data)\n", "# Our vectorized labels\n", "y_train = np.asarray(train_labels).astype('float32')\n", "y_test = np.asarray(test_labels).astype('float32')" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [], "source": [ "original_model = models.Sequential()\n", "original_model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "original_model.add(layers.Dense(16, activation='relu'))\n", "original_model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "original_model.compile(optimizer='rmsprop',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])\n", "\n", "smaller_model = models.Sequential()\n", "smaller_model.add(layers.Dense(4, activation='relu', input_shape=(10000,)))\n", "smaller_model.add(layers.Dense(4, activation='relu'))\n", "smaller_model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "smaller_model.compile(optimizer='rmsprop',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])\n", "original_hist = original_model.fit(x_train, y_train,\n", " epochs=20,\n", " batch_size=512, verbose=0,\n", " validation_data=(x_test, y_test))\n", "smaller_model_hist = smaller_model.fit(x_train, y_train,\n", " epochs=20,\n", " batch_size=512, verbose=0,\n", " validation_data=(x_test, y_test))" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "The smaller model starts overfitting later than the original one, and it overfits more _slowly_" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "epochs = range(1, 21)\n", "original_val_loss = original_hist.history['val_loss']\n", "smaller_model_val_loss = smaller_model_hist.history['val_loss']" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZwU1Z3v8c8PBGZ5UFRGg/IwiBgBBxQGlEuMGETRXQcfkoiXV3YhMdxgVOJmveriaxk17jW5Jt7oaiImBl0R14ercrMmGiNIdNUMsIACy4MEcJDoSAREQAb83T+qZmyG7pme6a5+qu/79epXd52uqv5N0dSv65xT55i7IyIi8dUh3wGIiEh+KRGIiMScEoGISMwpEYiIxJwSgYhIzB2R7wDaqlevXl5RUZHvMEREisrSpUs/dPfyZO8VXSKoqKhgyZIl+Q5DRKSomNnmVO+pakhEJOaUCEREYk6JQEQk5oqujSCZhoYG6urq2LdvX75DkTSVlZXRp08fOnXqlO9QRGKvJBJBXV0dPXr0oKKiAjPLdzjSCndn+/bt1NXVMWDAgHyHIxJ7JVE1tG/fPo499lglgSJhZhx77LG6ghNpo5qaaPZbEokAUBIoMvr3Emm7W2+NZr8lkwhERKR9IksEZvaQmX1gZm+neN/M7B4z22BmK81sRFSx5EJdXR2TJk1i0KBBDBw4kJkzZ7J///6k67733nt89atfbXWfF110ETt27GhXPDU1Ndx1113t2jZdc+fO5Zprrsl4HRFJraYGzIIHfP46m9VEUV4RzAUmtvD+hcCg8DEd+FmEsSSVrQPp7lx22WVccsklrF+/nnXr1rF7925mzZp12LoHDhzghBNO4Kmnnmp1v88//zw9e/bMTpAiUpRqasA9eMDnr4siEbj7YuAvLawyCXjEA28APc2sd1TxJJOt+raXX36ZsrIypk2bBkDHjh25++67eeihh9izZw9z586lurqar3zlK4wfP55NmzZx2mmnAbBnzx6+/vWvM2TIEC699FLOPPPMpiE0Kioq+PDDD9m0aRODBw/m29/+NkOHDuX8889n7969ADz44IOMGjWK4cOHc/nll7Nnz54WY506dSozZszgrLPO4qSTTmLRokV885vfZPDgwUydOrVpvfnz51NZWclpp53GjTfe2FT+q1/9ilNOOYXRo0fz2muvNZXX19dz+eWXM2rUKEaNGnXIeyJS2PLZRnAi8G7Ccl1YVnRWrVrFyJEjDyk78sgj6devHxs2bABg2bJlPPXUU7zyyiuHrHf//fdz9NFHs3r1am6//XaWLl2a9DPWr1/Pd7/7XVatWkXPnj15+umnAbjsssuora1lxYoVDB48mF/+8petxvvRRx/x+uuvc/fdd1NdXc3111/PqlWreOutt1i+fDnvvfceN954Iy+//DLLly+ntraWZ599lm3btjF79mxee+01Xn31VVavXt20z5kzZ3L99ddTW1vL008/zVVXXdWmYygirZs9O5r9FsV9BGY2naD6iH79+mW0r5qaQ68EGuvdZs+OrmsWwIQJEzjmmGMOK3/11VeZOXMmAKeddhrDhg1Luv2AAQM4/fTTARg5ciSbNm0C4O233+aWW25hx44d7N69mwsuuKDVWC6++GLMjMrKSo4//ngqKysBGDp0KJs2bWLz5s2MGzeO8vJgoMIpU6awePFigEPKr7jiCtatWwfASy+9dEhi2LVrF7t37241FhFJX1TnqHwmgq1A34TlPmHZYdx9DjAHoKqqyjP50Jqazw+m2ef1bpkYMmTIYXX+u3btYsuWLZx88sksW7aMbt26ZfQZXbp0aXrdsWPHpqqhqVOn8uyzzzJ8+HDmzp3LokWL0t5Xhw4dDtlvhw4dOHDgQLvu9v3ss8944403KCsra/O2IpJf+awaWgD8bdh76Cxgp7tvy2M87TZ+/Hj27NnDI488AsDBgwf5/ve/z9SpU+natWuL244dO5YnnngCgNWrV/PWW2+16bM//vhjevfuTUNDA/PmzWvfH9DM6NGjeeWVV/jwww85ePAg8+fP55xzzuHMM8/klVdeYfv27TQ0NPDkk082bXP++edz7733Ni0vX748K7GISPSi7D46H3gd+KKZ1ZnZt8zsO2b2nXCV54GNwAbgQeDqqGJJJVv1bWbGM888w5NPPsmgQYM45ZRTKCsr45//+Z9b3fbqq6+mvr6eIUOGcMsttzB06FCOOuqotD/79ttv58wzz2Ts2LGceuqpmfwZTXr37s2dd97Jueeey/Dhwxk5ciSTJk2id+/e1NTUMGbMGMaOHcvgwYObtrnnnntYsmQJw4YNY8iQIfz85z/PSiwiEj3zbNSN5FBVVZU3n5hmzZo1h5yUisnBgwdpaGigrKyMd955h/POO4+1a9fSuXPnfIcWuWL+dxMpNma21N2rkr1XFI3FpWzPnj2ce+65NDQ04O7cf//9sUgCIlI4lAjyrEePHpp6U0TySmMNiYjEnBKBiEjMKRGIiMScEoGISMwpEWTJHXfcwdChQxk2bBinn346b775Zlb22717d4BDBqorBOPGjWu1kTuddUQk/2KZCObNg4oK6NAheM70htzXX3+dX//61yxbtoyVK1fy0ksv0bdv39Y3jNCBAwfy+vkiUjxilwjmzYPp02Hz5mCcoc2bg+VMksG2bdvo1atX07g9vXr14oQTTgCCoaRvvvlmTj/9dKqqqli2bBkXXHABAwcObLr7dvfu3YwfP54RI0ZQWVnJc8891+LnHTx4kBtuuIFRo0YxbNgwHnjgAQAWLVrE2WefTXV1NUOGDDlsu+7du3PDDTcwdOhQzjvvPP74xz8ybtw4TjrpJBYsWAAE8z9PmzaNyspKzjjjDBYuXAjA3r17mTx5MoMHD+bSSy9tGusI4MUXX2TMmDGMGDGCr33taxpsTqTYuHtRPUaOHOnNrV69+rCyVPr3b5zW4dBH//5p7+IwH3/8sQ8fPtwHDRrkM2bM8EWLFiV8Xn+///773d39e9/7nldWVvquXbv8gw8+8OOOO87d3RsaGnznzp3u7l5fX+8DBw70zz77zN3du3Xr5u7uf/rTn3zo0KHu7v7AAw/47bff7u7u+/bt85EjR/rGjRt94cKF3rVrV9+4cWPSOAF//vnn3d39kksu8QkTJvj+/ft9+fLlPnz4cHd3v+uuu3zatGnu7r5mzRrv27ev792713/84x83la9YscI7duzotbW1Xl9f72effbbv3r3b3d3vvPNOv/XWW93d/ZxzzvHa2tqUx60t/24ikhlgiac4r8buhrItW9pWno7u3buzdOlS/vCHP7Bw4UKuuOIK7rzzzqaJXqqrqwGorKxk9+7d9OjRgx49etClSxd27NhBt27d+Md//EcWL15Mhw4d2Lp1K++//z5f+MIXkn7eiy++yMqVK5tGPN25cyfr16+nc+fOjB49mgEDBiTdrnPnzkycOLEpli5dutCpUycqKyubhrV+9dVXufbaawE49dRT6d+/P+vWrWPx4sVcd911AAwbNqxpuOw33niD1atXM3bsWAD279/PmDFj2n8wRSTnYpcI+vULqoOSlWeiY8eOjBs3jnHjxlFZWcnDDz/clAhaG/Z53rx51NfXs3TpUjp16kRFRQX79u1L+Vnuzr333nvY3AOLFi1qcbjrTp06YeEEDImxNMbRHu7OhAkTmD9/fru2F5H8i10bwR13QPORobt2Dcrba+3ataxfv75pefny5fTv3z/t7Xfu3Mlxxx1Hp06dWLhwIZuTZaoEF1xwAT/72c9oaGgAYN26dXzyySftC76Zs88+u2k463Xr1rFlyxa++MUv8uUvf5nHHnsMCCbDWblyJQBnnXUWr732WtNMbJ988knTZDUiUhxid0UwZUrwPGtWUB3Ur1+QBBrL22P37t1ce+217NixgyOOOIKTTz6ZOXPmtCGmKVx88cVUVlZSVVXV6nDSV111FZs2bWLEiBG4O+Xl5Tz77LPt/wMSXH311cyYMYPKykqOOOII5s6dS5cuXZgxYwbTpk1j8ODBDB48uGlqzvLycubOncuVV17Jp59+CsAPfvADTjnllKzEIyLR0zDUkjf6dxPJnZaGoY5d1ZCIiBxKiUBEJOZKJhEUWxVX3OnfS6RwRJoIzGyima01sw1mdlOS9/ub2e/NbKWZLTKzPu35nLKyMrZv366TS5Fwd7Zv305ZWVm+QxERIuw1ZGYdgfuACUAdUGtmC9x9dcJqdwGPuPvDZvYV4H8B32jrZ/Xp04e6ujrq6+uzEbrkQFlZGX36tCvvi0iWRdl9dDSwwd03ApjZ48AkIDERDAH+Pny9EGhXH8hOnTqlvJtWRERaFmXV0InAuwnLdWFZohXAZeHrS4EeZnZshDGJiEgz+W4s/gfgHDP7T+AcYCtwsPlKZjbdzJaY2RJV/4iIZFeUiWArkDgof5+wrIm7v+ful7n7GcCssGxH8x25+xx3r3L3qvLy8ghDFhGJnygTQS0wyMwGmFlnYDKwIHEFM+tlZo0x3Aw8FGE8IiKSRGSJwN0PANcALwBrgCfcfZWZ3WZm1eFq44C1ZrYOOB7IYOg3ERFpj5IYa0hERFqmsYZERCQlJQIRkZhTIhARiTklAhGRmFMiEBGJOSUCEZGYUyIQEYk5JQIRkZhTIhARiTklAhGRmFMiEBGJOSUCEZGYUyIQEYk5JQIRkZhTIhARiTklAhGRmFMiEBGJOSUCEZGYizQRmNlEM1trZhvM7KYk7/czs4Vm9p9mttLMLooyHhEROVxkicDMOgL3ARcCQ4ArzWxIs9VuIZjU/gxgMnB/VPGIiEhyUV4RjAY2uPtGd98PPA5MaraOA0eGr48C3oswHhERSeKICPd9IvBuwnIdcGazdWqAF83sWqAbcF6E8YiISBL5biy+Epjr7n2Ai4B/NbPDYjKz6Wa2xMyW1NfX5zxIEZFSFmUi2Ar0TVjuE5Yl+hbwBIC7vw6UAb2a78jd57h7lbtXlZeXRxSuiEg8RZkIaoFBZjbAzDoTNAYvaLbOFmA8gJkNJkgE+skvIpJDkSUCdz8AXAO8AKwh6B20ysxuM7PqcLXvA982sxXAfGCqu3tUMYmIyOGibCzG3Z8Hnm9W9k8Jr1cDY6OMQUREWpbvxmIRkdioqcl3BMm1mgjMrFtjTx4zO8XMqs2sU/ShiYiUlltvzXcEyaVzRbAYKDOzE4EXgW8Ac6MMSkREciedRGDuvge4DLjf3b8GDI02LBGR0lBTA2bBAz5/XUjVRGklAjMbA0wB/j0s6xhdSCIipaOmBtyDB3z+utgSwfeAm4Fnwu6fJwELow1LRORwhXTyLCWtJgJ3f8Xdq939h2Gj8Yfufl0OYhMROUShNrama/bsfEeQXDq9hh4zsyPNrBvwNrDazG6IPjQRkdJSqFc06VQNDXH3XcAlwG+AAQQ9h0REIlcMja3FLp1E0Cm8b+ASYIG7NxDMIyAiErlCamwt1eSTTiJ4ANhEMF/AYjPrD+yKMigRkUJU7G0UqbQ61pC73wPck1C02czOjS4kEZHkCrWxtdil01h8lJn9pHFiGDP7McHVgYhITuWrOqjU2yistVGfzexpgt5CD4dF3wCGu/tlEceWVFVVlS9ZsiQfHy0iMWf2eVtFsTGzpe5eley9dIahHujulycs32pmy7MTmoiI5Fs6jcV7zexLjQtmNhbYG11IIiKFqVTbKNK5IpgBPGxmRwEG/AWYGmVQIiKFqJTaBRKl02toOTDczI4Ml9V1VESkhKRMBGb29ynKAXD3n7S2czObCPyUYLTSX7j7nc3evxto7IraFTjO3XumFbmIiGRFS1cEPTLZsZl1BO4DJgB1QK2ZLQjnKQbA3a9PWP9a4IxMPlNERNouZSJw90zvoRsNbHD3jQBm9jgwCVidYv0rgRJtihERKVxRTl5/IvBuwnJdWHaYcNiKAcDLEcYjIiJJRJkI2mIy8JS7H0z2pplNb7yzub6+PsehiUipKNVeP5mKMhFsBfomLPcJy5KZDMxPtSN3n+PuVe5eVV5ensUQRSROSnXQuEy12n3UzLoAlwMVieu7+22tbFoLDDKzAQQJYDLw35Ps/1TgaOD1tKMWEZGsSeeK4DmCRt4DwCcJjxa5+wHgGuAFYA3wRDjn8W1mVp2w6mTgcW9t0CMRkXaIw6BxmUpn0Lm33f20HMXTKg06JyLtVcyDxmWqpUHn0rki+A8zq8xyTCIiUiDSGWvoS8BUM/sT8CnBeEPu7sMijUxEJMtKddC4TKWTCC6MPAoRkRxQu0ByrVYNuftmoCdwcfjoGZaJiEgJSGeqypnAPOC48PFoOC6QiIiUgHSqhr4FnOnunwCY2Q8J+vzfG2VgIiKSG+n0GjIgceiHg2GZiIiUgHSuCH4FvGlmz4TLlwC/jC4kERHJpXRmKPuJmS0i6EYKMM3d/zPSqEREJGdamqHsSHffZWbHAJvCR+N7x7j7X6IPT0REotbSFcFjwN8AS4HEm7ItXD4pwrhERCRHWpqh7G/C5wG5C0dERHItnfsIfp9OmYiIFKeW2gjKgK5ALzM7ms+7jB5JiiknRUSk+LR0RfA/CNoHTg2fGx/PAf8SfWgiUmo01k9hSmc+gmvdvWDuItZ8BCLFK87zAeRbS/MRpHMfwb1mdhowBChLKH8keyGKiEi+pNNYPJtgXKF7gXOBHwHVLW4kIhLSVJGFL52xhr4KjAf+7O7TgOHAUens3MwmmtlaM9tgZjelWOfrZrbazFaZ2WNpRy4iRaGmJqgOaqwSanytRFA40hlraK+7f2ZmB8zsSOADoG9rG5lZR+A+YAJQB9Sa2QJ3X52wziDgZmCsu39kZse1668QEZF2SycRLDGznsCDBL2GdhMMQ92a0cAGd98IYGaPA5OA1QnrfBu4z90/AnD3D9oQu4gUGU0VWZjSaSy+Onz5czP7LXCku69MY98nAu8mLNcBZzZb5xQAM3sN6AjUuPtvm+/IzKYD0wH69euXxkeLSCFSdVBhaumGshEtvefuy7L0+YOAcUAfYLGZVbr7jsSV3H0OMAeC7qNZ+FwREQm1dEXw4/C5DKgCVhDcXTwMWAKMaWXfWzm0LaFPWJaoDnjT3RuAP5nZOoLEUJtW9CIikrGUvYbc/Vx3PxfYBoxw9yp3HwmcweEn9GRqgUFmNsDMOgOTgQXN1nmW4GoAM+tFUFW0sc1/hYiItFs63Ue/6O5vNS64+9vA4NY2cvcDwDXAC8Aa4Al3X2Vmt5lZ430ILwDbzWw1sBC4wd23t/WPEBGR9ktniIn5wCfAo2HRFKC7u18ZcWxJaYgJkfypqVGDb7FqaYiJdK4IpgGrgJnhY3VYJiIxc+ut+Y5AopBO99F9wN3hQ0RESkzKKwIzeyJ8fsvMVjZ/5C5EEcknjRVU+lK2EZhZb3ffZmb9k73v7psjjSwFtRGI5I+GkS5e7RqG2t23hc95OeGLiEhutFQ19LGZ7Ury+NjMduUySBHJjkyrczRWUGlqtftooVHVkEj7qWonvjKaoSxhJ8dx6AxlW7IQm4iI5Fk6M5RVm9l64E/AK8Am4DcRxyUiWaJeP9KadG4oux04C1jn7gMIZit7I9KoRCRrNEOYtCadRNAQjv/Twcw6uPtCgtFIRUSkBKTTRrDDzLoDi4F5ZvYBwdhDIlJk1OtHkknnimASsBe4Hvgt8A5wcZRBiUg0VB0kybR0H8F9ZjbW3T9x94PufsDdH3b3ezRUtEh+6EQuUWjpimAdcJeZbTKzH5nZGbkKSkSS0+ifEoWWZij7qbuPAc4BtgMPmdl/mdlsMzslZxGKiMTcvHlQUQEdOgTP8+Zld/+tthG4+2Z3/6G7nwFcCVxCMOOYiOSA7gOIt3nzYPp02Lw56Pa7eXOwnM1kkM4MZUcAFxLMOTweWATMd/fnshdG+jTEhMSZhoiIn4qK4OTfXP/+sGlT+vtp1wxlZjbBzB4C6oBvA/8ODHT3yekmATObaGZrzWyDmd2U5P2pZlZvZsvDx1Xp/UkiIvGwJcVgPqnK26Ol+whuBh4Dvu/uH7V1x2bWEbgPmECQTGrNbIG7r2626r+5+zVt3b9IHOk+gPjp1y/5FUG/ftn7jJYai7/i7r9oTxIIjQY2uPtGd98PPE5wT4KItJPaBeLnjjuga9dDy7p2DcqzJZ0bytrrRODdhOW6sKy5y8PpL58ys77JdmRm081siZktqa+vjyJWEZHIZNLrZ8oUmDMnaBMwC57nzAnKsyXKRJCO/wdUuPsw4HfAw8lWcvc57l7l7lXl5eU5DVBEJBPZ6PUzZUrQMPzZZ8FzNpMARJsItgKJv/D7hGVN3H27u38aLv4CGBlhPCIiOTdrFuzZc2jZnj1BeaGIMhHUAoPMbICZdSbofrogcQUz652wWI3uT5ASpvr9eMpFr59MRZYI3P0AcA3wAsEJ/gl3X2Vmt5lZdbjadWa2ysxWANcBU6OKRyTfNDxEPKXq3ZPNXj+Z0pzFIjmim8HiqbGNILF6qGvX7Df4tqZdN5SJSOY0PERpKPReP5nSFYEUjZqa4j6B6oqgOBXKL/pMtXRFoEQgRaPYT6TFHn9cZWusn3xT1ZBIAdDwEMWpGHr9ZEqJQApaKdWxF2PMUhy9fjKlRCAFraYmqE5prFJpfK2TqrRFJo29uRjrJ9+UCETSpORTnDId4qEYev1kSo3FUjTy3WtIjb3FqVQaezOlxmIpCfpFLu0Rh8beTCkRiLSglBqr4yoOjb2ZUiIQaYEaq4tfHBp7M6VEICIFr9SHeMi3luYsFpEEuiEsP5oP8dDY6wfSP5lPmaITf0t0RSCxkWl1jqqD8qMYJnYpdkoEEhuaD6A4qddP9JQIRKSgqddP9JQIpKSp+2dh0BAPhU13Fkts6M7g/MjGeP7z5gVtAlu2BFcCd9yhxt+2ytt8BGY2Efgp0BH4hbvfmWK9y4GngFHu3uJZXolA2kuJID80xENhyMsQE2bWEbgPuBAYAlxpZkOSrNcDmAm8GVUsmVyWSulQ98/8UGNv4YuyjWA0sMHdN7r7fuBxYFKS9W4HfgjsiyKITEcelNKhdoH8UGNv4YsyEZwIvJuwXBeWNTGzEUBfd//3lnZkZtPNbImZLamvr29TEOqDLJI5NfaWtrz1GjKzDsBPgO+3tq67z3H3KnevKi8vb9Pn6LJUJDMaz7/0RZkItgJ9E5b7hGWNegCnAYvMbBNwFrDAzJI2ZrSXLktFMvtFn42r6ilTgobhzz4LnpUECkuUiaAWGGRmA8ysMzAZWND4prvvdPde7l7h7hXAG0B1a72G2kqXpRJ3mf6i11V16YssEbj7AeAa4AVgDfCEu68ys9vMrDqqz21Ol6USd5n+otdVdenTDWUiJa5Dh+T3T5gFVTWtycYNYZJ/mqpSJMYy/UWvq+rSp0QgUuKy0U6mxt7SpkQgOaMbuvJDv+ilNUoEkjOaD6D9Mh0mRb/opSWaqlKkwGVjqkaRluiKQCKl+QAyp2FSJGrqPio5o2Gg2yfT7p8ioO6jkiX6Fd8+mdbv64YuiZoSgaQt08beOM4HkI1h0DVMikRNiUByplivKAphwDZ1/5QoxSoRFOuJKJ/i3thbKAO2qfunRClWjcVqrMxMHI9fpvPtar5eKRRqLM6Q5jyOr0x/0at+X4pBySeCTKs2SmnO40yrc+LY2KsB2yQOVDXUilK6tI9j1Q4ESXvWrOBXfL9+wa/xdE/EGoJZSoWqhjKg2ZmKm+bbFWldrBJBe6o2iv1mnrj3+tF8uyKti1UiaM/Jr9gb+2pqgl/CjVVCja+LKRFk0livKzqR1kWaCMxsopmtNbMNZnZTkve/Y2ZvmdlyM3vVzIZEGU97qGogvzKt2in2KzqRXIgsEZhZR+A+4EJgCHBlkhP9Y+5e6e6nAz8CfhJVPJkolaqB9vb6yWf32Uyrdor9ik4kF6K8IhgNbHD3je6+H3gcmJS4grvvSljsBpRkn5ZsnUgzrc5pz/bZ6D6bz6odXdGJpMHdI3kAXwV+kbD8DeBfkqz3XeAd4F1gUIp9TQeWAEv69evnxeTRR927dm2smQ8eXbsG5W0F2Y+vNf37Hxp746N///S2z/Tvz/TzRSQALPEU5+u8Nxa7+33uPhC4EbglxTpz3L3K3avKy8tzG2CGin1SkUx/katqR6TwRZkItgJ9E5b7hGWpPA5cEmE8GWtP1UqmJ9J8d//MtLFVVTsihS/KRFALDDKzAWbWGZgMLEhcwcwGJSz+NbA+wngy1p7x+DM9kdbUwKOPBidACJ4ffbRtiSCTOvpMf5Fno9dOqTTWixSsVHVG2XgAFwHrCNoAZoVltwHV4eufAquA5cBCYGhr+xw5cmQ0FWhpaE8dfaZ15PnevnEf/fu7mwXPbd02W20kItJ+tNBGEGkiiOKR60Qwe3byxsrZs3OzfaaNpYXQ2JpJIhGR7GgpEcRq0LlMZTpoW3u2z3Tick18LiKgQeeKWqZ17LqzVkRao0TQBpmOx9+e7TNtrFX3SxFpjRJBG+Tjzt5Mu0+q+6WItEZtBCIiMaA2AhERSUmJQEQk5pQIRERiTolARCTmlAhERGKu6HoNmVk9sDnfcaTQC/gw30G0QPFlptDjg8KPUfFlJpP4+rt70nH8iy4RFDIzW5Kqe1YhUHyZKfT4oPBjVHyZiSo+VQ2JiMScEoGISMwpEWTXnHwH0ArFl5lCjw8KP0bFl5lI4lMbgYhIzOmKQEQk5pQIRERiTomgjcysr5ktNLPVZrbKzGYmWWecme00s+Xh459yHOMmM3sr/OzDhmq1wD1mtsHMVprZiBzG9sWE47LczHaZ2fearZPz42dmD5nZB2b2dkLZMWb2OzNbHz4fnWLbvwvXWW9mf5ej2P63mf1X+O/3jJn1TLFti9+FiGOsMbOtCf+OF6XYdqKZrQ2/jzflML5/S4htk5ktT7FtpMcw1Tklp9+/VHNY6pFikmfoDYwIX/cA1gFDmq0zDvh1HmPcBPRq4f2LgN8ABpwFvJmnODsCfya40SWvxw/4MjACeDuh7EfATeHrm4AfJtnuGGBj+Hx0+ProHMR2PnBE+PqHyWJL57sQcYw1wD+k8R14BzgJ6AysaP7/Kar4mr3/Y+Cf8nC4bHIAAAT5SURBVHEMU51Tcvn90xVBG7n7NndfFr7+GFgDnJjfqNpsEvCIB94AeppZ7zzEMR54x93zfqe4uy8G/tKseBLwcPj6YeCSJJteAPzO3f/i7h8BvwMmRh2bu7/o7gfCxTeAPtn8zLZKcfzSMRrY4O4b3X0/8DjBcc+qluIzMwO+DszP9uemo4VzSs6+f0oEGTCzCuAM4M0kb48xsxVm9hszG5rTwMCBF81sqZlNT/L+icC7Cct15CeZTSb1f758Hr9Gx7v7tvD1n4Hjk6xTCMfymwRXeMm09l2I2jVh9dVDKao2CuH4nQ287+7rU7yfs2PY7JySs++fEkE7mVl34Gnge+6+q9nbywiqO4YD9wLP5ji8L7n7COBC4Ltm9uUcf36rzKwzUA08meTtfB+/w3hwHV5wfa3NbBZwAJiXYpV8fhd+BgwETge2EVS/FKIraflqICfHsKVzStTfPyWCdjCzTgT/YPPc/f82f9/dd7n77vD180AnM+uVq/jcfWv4/AHwDMHld6KtQN+E5T5hWS5dCCxz9/ebv5Hv45fg/cYqs/D5gyTr5O1YmtlU4G+AKeGJ4jBpfBci4+7vu/tBd/8MeDDFZ+f1u2hmRwCXAf+Wap1cHMMU55Scff+UCNoorE/8JbDG3X+SYp0vhOthZqMJjvP2HMXXzcx6NL4maFR8u9lqC4C/DXsPnQXsTLgEzZWUv8LyefyaWQA09sL4O+C5JOu8AJxvZkeHVR/nh2WRMrOJwP8Eqt19T4p10vkuRBljYrvTpSk+uxYYZGYDwqvEyQTHPVfOA/7L3euSvZmLY9jCOSV337+oWsJL9QF8ieASbSWwPHxcBHwH+E64zjXAKoIeEG8A/y2H8Z0Ufu6KMIZZYXlifAbcR9Bb4y2gKsfHsBvBif2ohLK8Hj+CpLQNaCCoZ/0WcCzwe2A98BJwTLhuFfCLhG2/CWwIH9NyFNsGgrrhxu/gz8N1TwCeb+m7kMPj96/h92slwUmtd/MYw+WLCHrKvBNVjMniC8vnNn7vEtbN6TFs4ZySs++fhpgQEYk5VQ2JiMScEoGISMwpEYiIxJwSgYhIzCkRiIjEnBKBSMjMDtqhI6NmbSRMM6tIHPlSpJAcke8ARArIXnc/Pd9BiOSarghEWhGOR/+jcEz6P5rZyWF5hZm9HA6q9nsz6xeWH2/BHAErwsd/C3fV0cweDMecf9HM/ipc/7pwLPqVZvZ4nv5MiTElApHP/VWzqqErEt7b6e6VwL8A/ycsuxd42N2HEQz6dk9Yfg/wigeD5o0guCMVYBBwn7sPBXYAl4flNwFnhPv5TlR/nEgqurNYJGRmu929e5LyTcBX3H1jODjYn939WDP7kGDYhIawfJu79zKzeqCPu3+asI8KgnHjB4XLNwKd3P0HZvZbYDfBKKvPejjgnkiu6IpAJD2e4nVbfJrw+iCft9H9NcHYTyOA2nBETJGcUSIQSc8VCc+vh6//g2C0TIApwB/C178HZgCYWUczOyrVTs2sA9DX3RcCNwJHAYddlYhESb88RD73V3boBOa/dffGLqRHm9lKgl/1V4Zl1wK/MrMbgHpgWlg+E5hjZt8i+OU/g2Dky2Q6Ao+GycKAe9x9R9b+IpE0qI1ApBVhG0GVu3+Y71hEoqCqIRGRmNMVgYhIzOmKQEQk5pQIRERiTolARCTmlAhERGJOiUBEJOb+P1+6jR2PD65sAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot(epochs, original_val_loss, 'b+', label='Original model')\n", "plt.plot(epochs, smaller_model_val_loss, 'bo', label='Smaller model')\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Validation loss')\n", "plt.legend()\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Use 16 hidden nodes in the layers again, but now add weight regularization. Use L2 loss with alpha=0.001. What do you observe?" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "from keras import regularizers\n", "\n", "l2_model = models.Sequential()\n", "l2_model.add(layers.Dense(16, kernel_regularizer=regularizers.l2(0.001),\n", " activation='relu', input_shape=(10000,)))\n", "l2_model.add(layers.Dense(16, kernel_regularizer=regularizers.l2(0.001),\n", " activation='relu'))\n", "l2_model.add(layers.Dense(1, activation='sigmoid'))" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "l2_model.compile(optimizer='rmsprop',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "l2_model_hist = l2_model.fit(x_train, y_train,\n", " epochs=20,\n", " batch_size=512, verbose=0,\n", " validation_data=(x_test, y_test))" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "L2 regularized model is much more resistant to overfitting, even though both have the same number of parameters" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5wU9Znv8c/DCE4weB8JijBoUBluIiNojBfEeyIIXqLLyWYkkajBW7LeFldGPeYYT1Z3dTUJKkEXvGJUkpBojApHg7sMCshluegCjrI6EBERiTPwnD+qZmiG7pme6a6u7unv+/WqV3f96vZQ1NTT9atf/crcHRERKV6d4g5ARETipUQgIlLklAhERIqcEoGISJFTIhARKXJ7xB1AWx144IFeXl4edxgiIgVlwYIFG9y9LNm0gksE5eXl1NTUxB2GiEhBMbO1qaapakhEpMgpEYiIFDklAhGRIldw9wiSqa+vp7a2lm3btsUdihSI0tJSevbsSefOneMORSR2HSIR1NbW0q1bN8rLyzGzuMORPOfubNy4kdraWvr06RN3OCKx6xBVQ9u2beOAAw5QEpC0mBkHHHCAriCl4FRXR7PeDpEIACUBaRMdL1KIbrstmvV2mEQgIiLtE1kiMLOpZvaxmS1JMd3M7D4zW21mi83smKhiyYXa2lpGjx5N3759Ofzww7nmmmv48ssvk8774YcfcsEFF7S6znPOOYdNmza1K57q6mp+/vOft2vZdE2bNo2JEydmPI+IpFZdDWbBADu/Z7OaKMorgmnAWS1MPxvoGw4TgF9EGEtS2dqR7s7YsWM577zzWLVqFStXrmTLli1MmjRpt3kbGho4+OCDmTlzZqvrnT17Nvvuu292ghSRglRdDe7BADu/F0QicPe5wF9bmGU08JgH3gT2NbMeUcWTTLbq21555RVKS0u59NJLASgpKeHee+9l6tSpbN26lWnTpjFq1ChOPfVURo4cyZo1axgwYAAAW7du5aKLLqKiooIxY8YwfPjwpi40ysvL2bBhA2vWrKFfv35cdtll9O/fnzPOOIMvvvgCgIceeohjjz2WwYMHc/7557N169YWY62qquKKK67guOOO47DDDuO1115j/Pjx9OvXj6qqqqb5nnjiCQYOHMiAAQO48cYbm8p//etfc8QRRzBs2DDeeOONpvK6ujrOP/98jj32WI499thdpolIfovzHsEhwPsJ47VhWcFZunQpQ4cO3aVs7733plevXqxevRqAt956i5kzZzJnzpxd5nvwwQfZb7/9WLZsGXfccQcLFixIuo1Vq1bxox/9iKVLl7Lvvvvy7LPPAjB27Fjmz5/PokWL6NevH4888kir8X7yySfMmzePe++9l1GjRnHdddexdOlS3nnnHRYuXMiHH37IjTfeyCuvvMLChQuZP38+zz//POvXr2fy5Mm88cYbvP766yxbtqxpnddccw3XXXcd8+fP59lnn+UHP/hBm/ahiLRu8uRo1lsQzxGY2QSC6iN69eqV0bqqq3e9Emisd5s8ObqmWQCnn346+++//27lr7/+Otdccw0AAwYMYNCgQUmX79OnD0cffTQAQ4cOZc2aNQAsWbKEW265hU2bNrFlyxbOPPPMVmM599xzMTMGDhxI9+7dGThwIAD9+/dnzZo1rF27llNOOYWysqCjwnHjxjF37lyAXcq/853vsHLlSgBefvnlXRLD5s2b2bJlS6uxiEj6ojpHxZkIPgAOTRjvGZbtxt2nAFMAKisrPZONVlfv3JlmO+vdMlFRUbFbnf/mzZtZt24dX//613nrrbfYa6+9MtrGnnvu2fS9pKSkqWqoqqqK559/nsGDBzNt2jRee+21tNfVqVOnXdbbqVMnGhoa2vW07Y4dO3jzzTcpLS1t87IiEq84q4ZmAX8fth46DvjU3dfHGE+7jRw5kq1bt/LYY48BsH37dn7yk59QVVVF165dW1z2hBNO4OmnnwZg2bJlvPPOO23a9meffUaPHj2or69nxowZ7fsHNDNs2DDmzJnDhg0b2L59O0888QQnn3wyw4cPZ86cOWzcuJH6+nqeeeaZpmXOOOMM7r///qbxhQsXZiUWEYlelM1HnwDmAUeaWa2Zfd/MLjezy8NZZgPvAauBh4Aro4ollWzVt5kZzz33HM888wx9+/bliCOOoLS0lJ/+9KetLnvllVdSV1dHRUUFt9xyC/3792efffZJe9t33HEHw4cP54QTTuCoo47K5J/RpEePHtx1112MGDGCwYMHM3ToUEaPHk2PHj2orq7m+OOP54QTTqBfv35Ny9x3333U1NQwaNAgKioq+OUvf5mVWEQkeubZqBvJocrKSm/+Yprly5fvclIqJNu3b6e+vp7S0lLeffddTjvtNFasWEGXLl3iDq3DK+TjRqStzGyBu1cmm1YQN4s7sq1btzJixAjq6+txdx588EElARHJKSWCmHXr1k2v3hSRWKmvIRGRIqdEICJS5JQIRESKnBKBiEiRUyLIkq9+9au7ld1zzz1UVFQwaNAgRo4cydq1a3MeV3u6o541axZ33XVXxts+5ZRTIr8RXlVV1WpPrunMI1LMijIRzJgB5eXQqVPwmaUHcnczZMgQampqWLx4MRdccAE33HBDq8s0NDREE0yaGhoaGDVqFDfddFOscYhI7hRdIpgxAyZMgLVrg36G1q4NxqNIBiNGjGjqYuK4446jtrY26XxVVVVcfvnlDB8+nBtuuIHPP/+c8ePHM2zYMIYMGcILL7wAtNxldeIVycyZM3fpUrpRqi6rm28/8WUyRx99dNPwla98hTlz5qSM74svvuDiiy+mX79+jBkzpqk/pObKy8u5+eabOfroo6msrOStt97izDPP5PDDD296Itnduf766xkwYAADBw7kqaeeaiqfOHEiRx55JKeddhoff/xx03oXLFjAySefzNChQznzzDNZv74geywRybmie45g0iRo3mX/1q1B+bhx0W33kUce4eyzz045vba2lr/85S+UlJTwj//4j5x66qlMnTqVTZs2MWzYME477TR+8YtfNHVZvWTJkqbeSNM1duxYLrvsMgBuueUWHnnkEa666qrdtj9t2rSmZRr7DPrtb3/L3XffzTe+8Q0mT56cNL5f/epXdO3aleXLl7N48WKOOSb1S+d69erFwoULue6666iqquKNN95g27ZtDBgwgMsvv5zf/OY3LFy4kEWLFrFhwwaOPfZYTjrpJObNm8eKFStYtmwZH330ERUVFYwfP576+nquuuoqXnjhBcrKynjqqaeYNGkSU6dObdM+EilGRZcI1q1rW3k2TJ8+nZqamt3eRZDowgsvpKSkBICXXnqJWbNmNdXtb9u2jXXr1qXdZXUqLXVZnbj95latWsX111/Pq6++SufOnVPGN3fuXK6++moABg0a1GJ8o0aNAmDgwIFs2bKFbt260a1bN/bcc082bdrE66+/ziWXXEJJSQndu3fn5JNPZv78+cydO7ep/OCDD+bUU08FYMWKFSxZsoTTTz8dCLru6NEjp+85EilYRZcIevUKqoOSlUfh5Zdf5s4772TOnDlNXT5PmjSJ3//+98DOX9yJ3VS7O88++yxHHnlk2tuxxhcrEJyYk2mpy+pU3WRv2bKFiy66iIceeqjpxNqe+JprrSvstnJ3+vfvz7x589odk0ixKrp7BHfeCc17hu7aNSjPtrfffpsf/vCHzJo1i4MOOighhjtZuHBhyq6azzzzTO6//34aOwR8++23gZa7rO7evTvLly9nx44dPPfcc0nX254uq8ePH8+ll17KiSee2Gp8J510Eo8//jgQXH0sXrw4rW0kc+KJJ/LUU0+xfft26urqmDt3LsOGDeOkk05qKl+/fj2vvvoqAEceeSR1dXVNiaC+vp6lS5e2e/sixaTorgga7wNMmhRUB/XqFSSBTO8PbN26lZ49ezaN//jHP2b27Nls2bKFCy+8EAjqxWfNmtXquv7pn/6Ja6+9lkGDBrFjxw769OnD7373O6688kq+973vUVFRwVFHHbVLl9V33XUX3/72tykrK6OysjLp28Eau6wuKytj+PDhfPbZZy3GsXbtWmbOnMnKlSub6toffvjhlPFdccUVXHrppfTr149+/frt9vrOthgzZgzz5s1j8ODBmBl33303X/va1xgzZgyvvPIKFRUV9OrVi+OPPx6ALl26MHPmTK6++mo+/fRTGhoauPbaa+nfv3+7YxApFuqGuoCoy+rsKpbjRgTUDXWHoS6rRSQKSgQFRF1Wi0gUOszN4kKr4pJ46XgR2SnSRGBmZ5nZCjNbbWa79VlgZr3N7M9mttjMXjOznsnW05rS0lI2btyoP25Ji7uzceNGSktL4w5FJC9EVjVkZiXAA8DpQC0w38xmufuyhNl+Djzm7o+a2anA/wG+29Zt9ezZk9raWurq6rIRuhSB0tLSXVp5iRSzKO8RDANWu/t7AGb2JDAaSEwEFcCPw++vAs+3Z0OdO3emT58+GYQqIlK8oqwaOgR4P2G8NixLtAgYG34fA3QzswMijElERJqJ+2bxPwAnm9nbwMnAB8D25jOZ2QQzqzGzGlX/iIhkV5SJ4APg0ITxnmFZE3f/0N3HuvsQYFJYtqn5itx9irtXuntlWVlZhCGLiBSfKBPBfKCvmfUxsy7AxcAu/SuY2YFm1hjDzYD6DBYRybHIEoG7NwATgReB5cDT7r7UzG43s1HhbKcAK8xsJdAdiKDrNxERaUmH6GtIRERa1lJfQ3HfLBYRkZgpEYiIFDklAhGRIqdEICJS5JQIRESKnBKBiEiRUyIQESlySgQiIkVOiUBEpMgpEYiIFDklAhGRIqdEICJS5JQIRESKnBKBiEiRUyIQESlySgQiIkVOiUBEpMgpEYiIFDklAhGRIhdpIjCzs8xshZmtNrObkkzvZWavmtnbZrbYzM6JMh4REdldZInAzEqAB4CzgQrgEjOraDbbLcDT7j4EuBh4MKp4REQkuSivCIYBq939PXf/EngSGN1sHgf2Dr/vA3wYYTwiIpJElIngEOD9hPHasCxRNfC/zKwWmA1clWxFZjbBzGrMrKauri6KWEVEilbcN4svAaa5e0/gHODfzWy3mNx9irtXuntlWVlZzoMUEenIokwEHwCHJoz3DMsSfR94GsDd5wGlwIERxiQiIs1EmQjmA33NrI+ZdSG4GTyr2TzrgJEAZtaPIBGo7kdEJIciSwTu3gBMBF4ElhO0DlpqZreb2ahwtp8Al5nZIuAJoMrdPaqYRERkd3tEuXJ3n01wEzix7NaE78uAE6KMQUQkX1RXB0O+afWKwMz2aryBa2ZHmNkoM+scfWgiIh3LbbfFHUFy6VQNzQVKzewQ4CXgu8C0KIMSEZHcSScRmLtvBcYCD7r7hUD/aMMSEekYqqvBLBhg5/d8qiJKKxGY2fHAOOD3YVlJdCGJiHQc1dXgHgyw83uhJYJrgZuB58JWP4cBr0YbloiI5EqricDd57j7KHf/WXjTeIO7X52D2EREdpFPv6LbY/LkuCNILp1WQ4+b2d5mthewBFhmZtdHH5qIyK7ytdVNuvI1kaVTNVTh7puB84A/AH0IWg6JiEgHkE4i6Bw+N3AeMMvd6wm6jxYRiVw+tbrJ11/0mbLWenQws6uBG4FFwLeAXsB0dz8x+vB2V1lZ6TU1NXFsWkRiZraz9U0xbj8TZrbA3SuTTWu1iwl3vw+4L6ForZmNyFZwIiISr3RuFu9jZvc0vhjGzP4Z2CsHsYmI7CKOVjf5VDUVlXSqhp4laC30aFj0XWCwu4+NOLakVDUkInEp2qoh4HB3Pz9h/DYzW5id0EREJG7ptBr6wsy+2ThiZicAX0QXkohIfsrXB8Iylc4VwRXAo2a2D2DAX4GqKIMSEclHHem+QKJ0Wg0tBAab2d7h+ObIoxIRkZxJmQjM7McpygFw93siiklERHKopSuCbpmu3MzOAv6VoNvqh939rmbT7wUan0noChzk7vtmul0REUlfykTg7hl172RmJcADwOlALTDfzGaF7ylu3MZ1CfNfBQzJZJsiItJ26bQaaq9hwGp3f8/dvwSeBEa3MP8lwBMRxiMiIklEmQgOAd5PGK8Ny3ZjZr0JejV9JcX0CY1PNtfV1WU9UBEpDh211U+mokwEbXExMNPdtyeb6O5T3L3S3SvLyspyHJqIdBSF/j6DqLTafNTM9gTOB8oT53f321tZ9APg0ITxnmFZMhcDP2otFhERyb50rgheIKjbbwA+TxhaMx/oa2Z9zKwLwcl+VvOZzOwoYD9gXrpBi4ikqxg6jctUOk8W93T3s9q6YndvMLOJwIsEzUenuvtSM7sdqHH3xqRwMfCkt9b7nYhIO1RX7zzpF3KncVFKJxH8xcwGuvs7bV25u88GZjcru7XZeHVb1ysiItmTTiL4JlBlZv8N/I2gvyF390GRRiYikmUdtdO4TKWTCM6OPAoRkRzQfYHkWr1Z7O5rgX2Bc8Nh37BMREQ6gHReVXkNMAM4KBymh91BiIhIB5BO1dD3geHu/jmAmf2MoKnn/VEGJiIiuZHOcwQGJD7xuz0sExGRDiCdK4JfA/9hZs+F4+cBj0QXkoiI5FI6byi7x8xeI2hGCnCpu78daVQiIpIzLb2hbG9332xm+wNrwqFx2v7u/tfowxMRkai1dEXwOPBtYAGQ+FC2heOHRRiXiIjkSEtvKPt2+Nknd+GIiEiupfMcwZ/TKRMRkcLU0j2CUoIXyh9oZvuxs8no3qR405iIiBSelq4Ifkhwf+Co8LNxeAH4t+hDE5GORn395Cdr7TUAZnaVu+fNU8SVlZVeU1MTdxgi0g56H0B8zGyBu1cmm5bOcwT3m9kAoAIoTSh/LHshiohIXNK5WTyZoF+h+4ERwN3AqIjjEpEOQq+KzH/pVA29AwwG3nb3wWbWHZju7qfnIsDmVDUkUrhUNRSflqqG0ul07gt33wE0mNnewMfAoWlu+CwzW2Fmq83sphTzXGRmy8xsqZk9ns56RUQke9LpdK7GzPYFHiJoNbSFoBvqFplZCfAAcDpQC8w3s1nuvixhnr7AzcAJ7v6JmR3Ujn+DiBQIvSoyP6Vzs/jK8OsvzeyPwN7uvjiNdQ8DVrv7ewBm9iQwGliWMM9lwAPu/km4rY/bEryIFBbdF8hPLT1QdkxL09z9rVbWfQjwfsJ4LTC82TxHhOt7AygBqt39j0m2NwGYANCrV69WNisiIm3R0hXBP4efpUAlsIjg6eJBQA1wfJa23xc4BegJzDWzge6+KXEmd58CTIHgZnEWtisiIqGUN4vdfYS7jwDWA8e4e6W7DwWGAB+kse4P2PWmcs8ky9UCs9y93t3/G1hJkBhERCRH0mk1dKS7v9M44u5LgH5pLDcf6GtmfcysC3AxMKvZPM8TXA1gZgcSVBW9l8a6RUQkS9JJBIvN7GEzOyUcHgJavVns7g3AROBFYDnwtLsvNbPbzazxgbQXgY1mtgx4Fbje3Te2758iIlHTzd6OKZ0HykqBK4CTwqK5wC/cfVvEsSWlB8pE4qMHwgpXpn0NbQPuDQcREelgUlYNmdnT4ec7Zra4+ZC7EEUkTuorqONLWTVkZj3cfb2Z9U423d3XRhpZCqoaEomPqoYKV7uqhtx9ffgZywlfRERyo6Wqoc/MbHOS4TMz25zLIEUkOzKtzlFfQR1Tq62G8o2qhkTaT1U7xSujVkMJKzmIXd9Qti4LsYmISMzSeUPZKDNbBfw3MAdYA/wh4rhEJEvU6kdak86TxXcAxwEr3b0PMBJ4M9KoRCRrqquD6qDGKqHG70oE0iidRFAfdvvQycw6ufurBL2RiohIB5DOPYJNZvZVgq4lZpjZx8Dn0YYlIlFQqx9JJp0rgtHAF8B1wB+Bd4FzowxKRKKh6iBJpqXnCB4wsxPc/XN33+7uDe7+qLvfpx5CReKhE7lEoaUrgpXAz81sjZndbWZDchWUiCR3221xRyAdUUtvKPtXdz8eOBnYCEw1s/8ys8lmdkTOIhQRkUi1eo/A3de6+8/cfQhwCXAewYtmRCQH9ByARC2dB8r2MLNzzWwGwYNkK4CxkUcmIoCeA5DopWw+amanE1wBnAP8J/AkMMHd1XRURKQDaemK4GbgL0A/dx/l7o+3NQmY2VlmtsLMVpvZTUmmV5lZnZktDIcftDF+kaKi5wAkCi29j+DUTFZsZiXAA8DpQC0w38xmufuyZrM+5e4TM9mWSLFQdZBEIZ0HytprGLDa3d9z9y8JqpZGR7g9ERFphygTwSHA+wnjtWFZc+eH70GeaWaHJluRmU0wsxozq6mrq4siVhGRohVlIkjHb4Fydx8E/Al4NNlM7j7F3SvdvbKsrCynAYqIdHRRJoIPgMRf+D3DsibuvtHd/xaOPgwMjTAeERFJIspEMB/oa2Z9zKwLcDEwK3EGM+uRMDoKPagmHZhu9Eq+iiwRuHsDMBF4keAE/7S7LzWz281sVDjb1Wa21MwWAVcDVVHFIxI39RMk+UovrxfJEb04XuLU0svr475ZLJK2QqxaUT9BUgh0RSAFo9B/URd6/FLYdEUgIiIpKRFIXutIVSvqJ0jylaqGpGCoakWk/VQ1JJIFhXgVIpIOJQIpGHFXreg5AOmolAikYOgXuUg0lAhEWtCRblaLpKJEINICvS+4Y5gxA8rLoVOn4HPGjLgjyi9KBCLSoc2YARMmwNq1QRJfuzYYL6RkEHUiUyIQSVPcN6ulfSZNgq1bdy3bujUoz5VMTuS5SGR6jkCKRnW1qnSKUadOyZ8/MYMdO6LffuOJPDEZde0KU6bAuHGtL19eHpz8m+vdG9asST8OPUcggpp/FqtevdpWnkwmv+gzvSJZt65t5e2hRCB5Tzf6JJNj4M47g1/gibp2DcrT3XYmVTOZnsizkcha5e4FNQwdOtSleEyf7t61a2NbnWDo2jUoT8fkybsu2zhMnhxl1JJNmR4Djevo3dvdLPhsy7K9eyc/hnr3zs3y2fj3u7sDNZ7ivBr7ib2tgxJBccn0j8h950mgcbm2/gFlKpOTUEeKob2ycQxkwiz59s3SWz7uRNZIiUAKVj78EWYi7u1nK4ZMT0SZLJ/pMZCpbP4YiTMRx5YIgLOAFcBq4KYW5jsfcKCytXUqERSXuC/LMxX39rMRQ6aJJNPl496H+ZDMsyGWRACUAO8ChwFdgEVARZL5ugFzgTejSgT5kI2lfTL9I4z712Tc289GDHEn43w4EXeEc0hcieB44MWE8ZuBm5PM9y/At4DXokgE+XAQSWbivNGXqbi3n40YMk0k2UiGHeFEHLeWEkGUzUcPAd5PGK8Ny5qY2THAoe7++5ZWZGYTzKzGzGrq6uraFEQ+PFVY7DJt/jluXPDgzI4dwWc6D+E0yrTpYKaytf04m09m2nwxG80fMzkGJA2pMkSmA3AB8HDC+HeBf0sY70RwFVAejr9GBFcE+XBpXszy4Yos7l+T2bjRGufN3rjvEUh2kI9VQ8A+wAZgTThsAz5sLRm0NRHkw6V5MdP+z1w+7MM4Ww1JdrSUCCLra8jM9gBWAiOBD4D5wN+5+9IU878G/IO7t9iRUFv7Gsq0nw/JTNz9vGTDjBlBVeK6dUF1xp135vbY6Qj7UOIXS19D7t4ATAReBJYDT7v7UjO73cxGRbXd5saNC076vXsHfzi9eysJ5FJOHo+PUD50YVzo+1Dyn3oflUgV+hVZtnp+zESh70PJD+p9VGJT6Fdkuej5sTWFvg8l/ykRpEG9X2amsenfrbcWXtO/fKmWUfNJiZISQSvyoY64oyjE9wHE/RyCSC4oEbRCD6QVN1XLSDFQImhFPtQRxy2TqrHq6uAEahaMN34vpFdGqlpGOjq1GmpFPrQaiVM2W6yYJW8PLyLRU6uhDHSEOuI437cqIvlPiaAVhV5HHPf7VhNNntz2ZUQkeqoa6uAyrdoq9qoxkY5CVUMxy/Q5hEyWz/QX/dChbSsXkcJTVIkgjpYqmVbNZLp8pg9EPfssTJ8eXAFA8Dl9elAuIh1DUVUNxdFqJe6qGbX6ERFQ1VCsMq2ayXT5xJvdkNnNbt3sFemYOnwiiPuBpnx6zR9k9kBUIT0EJiLpK4pE0PhOJ9j5PVcntUyfQ+gIzzGISH7r8Ikgbpk+h5Dp8nFfEYlI/iuqm8XV1cV9AtTNXpHipZvFoWJOAiIiqUSaCMzsLDNbYWarzeymJNMvN7N3zGyhmb1uZhVRxlPs1OpHRJKJLBGYWQnwAHA2UAFckuRE/7i7D3T3o4G7gXuiikd0RSQiyUV5RTAMWO3u77n7l8CTwOjEGdx9c8LoXoBqsFugE7mIRCHKRHAI8H7CeG1Ytgsz+5GZvUtwRXB1shWZ2QQzqzGzmrq6ukiCLQSF+KpHEcl/sd8sdvcH3P1w4EbglhTzTHH3SnevLCsry22AIiIdXJSJ4APg0ITxnmFZKk8C50UYT8YyrZppz/J6DkBEohbZcwRmtgewEhhJkADmA3/n7ksT5unr7qvC7+cCk1O1c20U5/sIMm2HH/fyIlK8WnqOYI+oNuruDWY2EXgRKAGmuvtSM7sdqHH3WcBEMzsNqAc+Ab4XVTwiIpJcpPcI3H22ux/h7oe7+51h2a1hEsDdr3H3/u5+tLuPSLxayBeZVs1ks2pHzwGISBSKqouJTKlqR0QKlbqYEBGRlJQI2iDTqhlV7YhIPlLVkIhIEVDVkIiIpKREICJS5JQIRESKnBKBiEiRUyIQESlyBddqyMzqgLVxx5HCgcCGuINogeLLTL7HB/kfo+LLTCbx9Xb3pN03F1wiyGdmVtNap3lxUnyZyff4IP9jVHyZiSo+VQ2JiBQ5JQIRkSKnRJBdU+IOoBWKLzP5Hh/kf4yKLzORxKd7BCIiRU5XBCIiRU6JQESkyCkRtJGZHWpmr5rZMjNbambXJJnnFDP71MwWhsOtOY5xjZm9E257t65aLXCfma02s8VmdkwOYzsyYb8sNLPNZnZts3lyvv/MbKqZfWxmSxLK9jezP5nZqvBzvxTLfi+cZ5WZZf11qyli+79m9l/h/99zZrZvimVbPBYijrHazD5I+H88J8WyZ5nZivB4vCmH8T2VENsaM1uYYtlI92Gqc1CWVNEAAAVxSURBVEpOjz9319CGAegBHBN+7wasBCqazXMK8LsYY1wDHNjC9HOAPwAGHAf8R0xxlgD/Q/CgS6z7DzgJOAZYklB2N3BT+P0m4GdJltsfeC/83C/8vl8OYjsD2CP8/rNksaVzLEQcYzXwD2kcA+8ChwFdgEXN/56iiq/Z9H8Gbo1jH6Y6p+Ty+NMVQRu5+3p3fyv8/hmwHDgk3qjabDTwmAfeBPY1sx4xxDESeNfdY39S3N3nAn9tVjwaeDT8/ihwXpJFzwT+5O5/dfdPgD8BZ0Udm7u/5O4N4eibQM9sbrOtUuy/dAwDVrv7e+7+JfAkwX7PqpbiMzMDLgKeyPZ209HCOSVnx58SQQbMrBwYAvxHksnHm9kiM/uDmfXPaWDgwEtmtsDMJiSZfgjwfsJ4LfEks4tJ/ccX5/5r1N3d14ff/wfonmSefNiX4wmu8JJp7ViI2sSw+mpqiqqNfNh/JwIfufuqFNNztg+bnVNydvwpEbSTmX0VeBa41t03N5v8FkF1x2DgfuD5HIf3TXc/Bjgb+JGZnZTj7bfKzLoAo4BnkkyOe//txoPr8Lxra21mk4AGYEaKWeI8Fn4BHA4cDawnqH7JR5fQ8tVATvZhS+eUqI8/JYJ2MLPOBP9hM9z9N82nu/tmd98Sfp8NdDazA3MVn7t/EH5+DDxHcPmd6APg0ITxnmFZLp0NvOXuHzWfEPf+S/BRY5VZ+Plxknli25dmVgV8GxgXnih2k8axEBl3/8jdt7v7DuChFNuO9Vg0sz2AscBTqebJxT5McU7J2fGnRNBGYX3iI8Byd78nxTxfC+fDzIYR7OeNOYpvLzPr1vid4KbikmazzQL+Pmw9dBzwacIlaK6k/BUW5/5rZhbQ2Arje8ALSeZ5ETjDzPYLqz7OCMsiZWZnATcAo9x9a4p50jkWoowx8b7TmBTbng/0NbM+4VXixQT7PVdOA/7L3WuTTczFPmzhnJK74y+qO+EddQC+SXCJthhYGA7nAJcDl4fzTASWErSAeBP4Rg7jOyzc7qIwhklheWJ8BjxA0FrjHaAyx/twL4IT+z4JZbHuP4KktB6oJ6hn/T5wAPBnYBXwMrB/OG8l8HDCsuOB1eFwaY5iW01QN9x4DP4ynPdgYHZLx0IO99+/h8fXYoKTWo/mMYbj5xC0lHk3qhiTxReWT2s87hLmzek+bOGckrPjT11MiIgUOVUNiYgUOSUCEZEip0QgIlLklAhERIqcEoGISJFTIhAJmdl227Vn1Kz1hGlm5Yk9X4rkkz3iDkAkj3zh7kfHHYRIrumKQKQVYX/0d4d90v+nmX09LC83s1fCTtX+bGa9wvLuFrwjYFE4fCNcVYmZPRT2Of+SmX0lnP/qsC/6xWb2ZEz/TCliSgQiO32lWdXQdxKmferuA4F/A/4lLLsfeNTdBxF0+nZfWH4fMMeDTvOOIXgiFaAv8IC79wc2AeeH5TcBQ8L1XB7VP04kFT1ZLBIysy3u/tUk5WuAU939vbBzsP9x9wPMbANBtwn1Yfl6dz/QzOqAnu7+t4R1lBP0G983HL8R6Ozu/9vM/ghsIehl9XkPO9wTyRVdEYikx1N8b4u/JXzfzs57dN8i6PvpGGB+2COmSM4oEYik5zsJn/PC738h6C0TYBzw/8LvfwauADCzEjPbJ9VKzawTcKi7vwrcCOwD7HZVIhIl/fIQ2ekrtusLzP/o7o1NSPczs8UEv+ovCcuuAn5tZtcDdcClYfk1wBQz+z7BL/8rCHq+TKYEmB4mCwPuc/dNWfsXiaRB9whEWhHeI6h09w1xxyISBVUNiYgUOV0RiIgUOV0RiIgUOSUCEZEip0QgIlLklAhERIqcEoGISJH7/+taQWfApks/AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "l2_model_val_loss = l2_model_hist.history['val_loss']\n", "\n", "plt.plot(epochs, original_val_loss, 'b+', label='Original model')\n", "plt.plot(epochs, l2_model_val_loss, 'bo', label='L2-regularized model')\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Validation loss')\n", "plt.legend()\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "* Add a drop out layer after every dense layer. Use a dropout rate of 0.5. What do you observe?" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "``` python\n", "dpt_model = models.Sequential()\n", "dpt_model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "dpt_model.add(layers.Dropout(0.5))\n", "dpt_model.add(layers.Dense(16, activation='relu'))\n", "dpt_model.add(layers.Dropout(0.5))\n", "dpt_model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "dpt_model.compile(optimizer='rmsprop',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])\n", "```" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "dpt_model = models.Sequential()\n", "dpt_model.add(layers.Dense(16, activation='relu', input_shape=(10000,)))\n", "dpt_model.add(layers.Dropout(0.5))\n", "dpt_model.add(layers.Dense(16, activation='relu'))\n", "dpt_model.add(layers.Dropout(0.5))\n", "dpt_model.add(layers.Dense(1, activation='sigmoid'))\n", "\n", "dpt_model.compile(optimizer='rmsprop',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "slideshow": { "slide_type": "skip" } }, "outputs": [], "source": [ "dpt_model_hist = dpt_model.fit(x_train, y_train,\n", " epochs=20,\n", " batch_size=512, verbose=0,\n", " validation_data=(x_test, y_test))" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "Dropout finds a better model, and overfits more slowly as well" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "slideshow": { "slide_type": "-" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3wU9b3/8deHgKUogpdoLZcEKKjchXDxoFK84gVQqRZ+1Iq2WlppqW39aY+eErWeY63VU8UbtoJW6gVaKb8erdoqUKy2gCdeQEWEIEGtEQVEtAb4/P6YSdwkm80mu7OX7Pv5eMxjd2ZnZz5ZlvnsfL/f+Yy5OyIiUrjaZTsAERHJLiUCEZECp0QgIlLglAhERAqcEoGISIFrn+0AWurggw/20tLSbIchIpJXVq9e/Z67F8d7Le8SQWlpKatWrcp2GCIiecXMNjX1mpqGREQKnBKBiEiBUyIQESlweddHEE9NTQ1VVVV88skn2Q5FpEkdO3ake/fudOjQIduhiNTTJhJBVVUVnTt3prS0FDPLdjgijbg7W7dupaqqil69emU7HJF62kTT0CeffMJBBx2kJCA5y8w46KCDdNYqKSkvj2a7bSIRAEoCkvP0HZVUXX11NNttM4lARERaJ7JEYGb3mNm7ZvZyE6+bmd1iZuvN7EUzGxZVLJlQVVXFpEmT6Nu3L3369GHWrFl8+umncdd96623+MpXvtLsNk877TS2bdvWqnjKy8u58cYbW/XeZM2fP5+ZM2emvI6INK28HMyCCT57ns5moijPCOYD4xO8firQN5wuBu6IMJa40vVBujtnn302Z555Jq+//jrr1q1j586dXHnllY3W3b17N1/84hdZtGhRs9t99NFH6dq1a3qCFJG8VF4O7sEEnz3Pi0Tg7suB9xOsMgm4zwPPAV3N7LCo4oknXe1tTz31FB07duSCCy4AoKioiJtvvpl77rmHXbt2MX/+fCZOnMjxxx/PCSecQGVlJQMHDgRg165dnHvuufTv35+zzjqLUaNG1ZXQKC0t5b333qOyspIjjzySiy66iAEDBnDyySfz8ccfA3D33XczYsQIhgwZwuTJk9m1a1fCWKdPn863v/1tRo8eTe/evVm6dCkXXnghRx55JNOnT69b74EHHmDQoEEMHDiQyy+/vG75vHnz6NevHyNHjuSZZ56pW15dXc3kyZMZMWIEI0aMqPeaiOS2bPYRdAM2x8xXhcvyzpo1axg+fHi9Zfvvvz89e/Zk/fr1ADz//PMsWrSIZcuW1Vvv9ttv54ADDmDt2rVce+21rF69Ou4+Xn/9dS655BLWrFlD165d+d3vfgfA2WefzcqVK3nhhRc48sgj+fWvf91svB988AHPPvssN998MxMnTuTSSy9lzZo1vPTSS1RUVPDWW29x+eWX89RTT1FRUcHKlStZvHgxb7/9NrNnz+aZZ55hxYoVrF27tm6bs2bN4tJLL2XlypX87ne/45vf/GaLPkMRad7s2dFsNy+uIzCziwmaj+jZs2dK2yovr38mUNvuNnt2dEOzAE466SQOPPDARstXrFjBrFmzABg4cCCDBw+O+/5evXoxdOhQAIYPH05lZSUAL7/8MldddRXbtm1j586dnHLKKc3GMmHCBMyMQYMGceihhzJo0CAABgwYQGVlJZs2beLLX/4yxcVBocJp06axfPlygHrLv/rVr7Ju3ToA/vznP9dLDDt27GDnzp3NxiIiyYvqGJXNRLAF6BEz3z1c1oi7zwXmApSVlXkqOy0v/+zDNPus3S0V/fv3b9Tmv2PHDt58802+9KUv8fzzz7PvvvumtI/Pfe5zdc+LiorqmoamT5/O4sWLGTJkCPPnz2fp0qVJb6tdu3b1ttuuXTt2797dqitf9+7dy3PPPUfHjh1b/F4Rya5sNg0tAb4ejh4aDWx397ezGE+rnXDCCezatYv77rsPgD179vDDH/6Q6dOn06lTp4TvHTNmDA8//DAAa9eu5aWXXmrRvj/88EMOO+wwampqWLBgQev+gAZGjhzJsmXLeO+999izZw8PPPAAY8eOZdSoUSxbtoytW7dSU1PDwoUL695z8sknc+utt9bNV1RUpCUWEYlelMNHHwCeBQ43syoz+4aZzTCzGeEqjwIbgPXA3cB3ooqlKelqbzMzHnnkERYuXEjfvn3p168fHTt25D//8z+bfe93vvMdqqur6d+/P1dddRUDBgygS5cuSe/72muvZdSoUYwZM4YjjjgilT+jzmGHHcb111/PuHHjGDJkCMOHD2fSpEkcdthhlJeXc/TRRzNmzBiOPPLIuvfccsstrFq1isGDB9O/f3/uvPPOtMQiItEzT0fbSAaVlZV5wxvTvPLKK/UOSvlkz5491NTU0LFjR9544w1OPPFEXnvtNfbZZ59shyYRyOfvquQ3M1vt7mXxXsuLzuK2bNeuXYwbN46amhrcndtvv11JQEQySokgyzp37qxbb4pIVqnWkIhIgVMiEBEpcEoEIiIFTolARKTAKRGkSVFREUOHDmXAgAEMGTKEX/ziF+zduzdr8SxevLheyYdM2m+//Vr8nlRKbtdaunQpZ5xxRkrbaE5swcBU1hHJJQWZCBYsgNJSaNcueEzHBbmf//znqaioYM2aNTz55JM89thjXB2nvOnu3btT31kSkk0E7p7VhFW7f5XcFsmegksECxbAxRfDpk1BnaFNm4L5NFVnAOCQQw5h7ty5zJkzB3dvVIba3bnssssYOHAggwYN4qGHHgKCX7THHXccp59+OocffjgzZsyoO0g3VRY69tf3okWLmD59On/7299YsmQJl112GUOHDuWNN96oF19lZSWHH344X//61xk4cCCbN2/miSee4Oijj2bYsGGcc845dQXjHn30UY444giGDx/O9773vbpf3A1vfDNw4MC6Qni1du7cyQknnMCwYcMYNGgQf/jDH5rcf23J7TvvvJOhQ4cydOhQevXqxbhx4wCajO9Pf/oTRxxxBMOGDeP3v/993H+P+fPnc+aZZ3LSSSdRWlrKnDlzuOmmmzjqqKMYPXo0778fVEuvqKhg9OjRDB48mLPOOosPPvgAgNWrVzNkyBCGDBnCbbfdVrfdPXv2cNlllzFixAgGDx7MXXfd1ex3QyQnuXteTcOHD/eG1q5d22hZU0pKam/rUH8qKUl6E3Htu+++jZZ16dLF33nnHZ83b55369bNt27d6u7uixYt8hNPPNF3797t77zzjvfo0cPfeustf/rpp/1zn/ucv/HGG757924/8cQTfeHChb5lyxbv0aOHv/vuu15TU+Pjxo3zRx55pNF+Fy5c6Oeff767u59//vm+cOHCuLFu3LjRzcyfffZZd3evrq72Y4891nfu3Onu7tdff71fffXV/vHHH3v37t19w4YN7u4+ZcoUP/30093dffbs2f7zn/+8bpsDBgzwjRs31ouppqbGt2/fXrePPn36+N69exvt3929pKTEq6ur6+Y//fRTP+aYY3zJkiXNxrdu3Trfu3evn3POOXXxxZo3b5736dPHd+zY4e+++67vv//+fscdd7i7+/e//32/+eab3d190KBBvnTpUnd3/4//+A+fNWtW3fJly5a5u/uPfvQjHzBggLu733XXXX7ttde6u/snn3ziw4cP9w0bNvjGjRvr1mmoJd9VkXQCVnkTx9WCOyN4882WLU+X2DLUK1asYOrUqRQVFXHooYcyduxYVq5cCQQF33r37k1RURFTp05lxYoVrFy5sq78c/v27euVhW6tkpISRo8eDcBzzz3H2rVrGTNmDEOHDuXee+9l06ZNvPrqq/Tu3ZtevXoBMHXq1Bbtw93593//dwYPHsyJJ57Ili1b+Oc//9lo//HMmjWL448/ngkTJiSMr1evXvTt2xcz42tf+1qT2xs3bhydO3emuLiYLl26MGHCBAAGDRpEZWUl27dvZ9u2bYwdOxaA888/n+XLl7Nt2za2bdvGcccdB8B5551Xt80nnniC++67j6FDhzJq1Ci2bt3K66+/3qLPSCQXFNyVxT17Bs1B8Zan04YNGygqKuKQQw4BSLoMtdXeIKGJ+UTrf/LJJ3HX2bx5c92Bb8aMGYwfP75ePO7OSSedxAMPPFDvfYkqiLZv375e30K8fS9YsIDq6mpWr15Nhw4dKC0trVsv0ecxf/58Nm3axJw5c1odX0MNy23HluJubb+Nu3Prrbc2ugdEwyYykVxXcGcE110HDStDd+oULE+X6upqZsyYwcyZM+MeyI899lgeeugh9uzZQ3V1NcuXL2fkyJEA/OMf/2Djxo3s3buXhx56iGOOOabJstAAhx56KK+88gp79+7lkUceqdtH586d+fDDDwHo0aMHFRUVVFRUMGPGjEbxjB49mmeeeabubmofffQR69at4/DDD2fDhg11B7bavgwIbqP5/PPPA8Hd1zZu3Nhou9u3b+eQQw6hQ4cOPP3002yKl4EbWL16NTfeeCP3338/7dq1SxjfEUccQWVlZV0fSMNE0RJdunThgAMO4K9//SsAv/nNbxg7dixdu3ala9eurFixAqBeqe9TTjmFO+64g5qaGgDWrVvHRx991OoYRLKl4M4Ipk0LHq+8MmgO6tkzSAK1y1vr448/ZujQodTU1NC+fXvOO+88fvCDH8Rd96yzzuLZZ59lyJAhmBk33HADX/jCF3j11VcZMWIEM2fOZP369YwbN46zzjqLdu3a1ZWFdndOP/10Jk2aBMD111/PGWecQXFxMWVlZXWdqFOmTOGiiy7illtuYdGiRfTp06fJ2IuLi5k/fz5Tp07lX//6FwA//elP6devH7fffnvdGcSIESPq3jN58mTuu+8+BgwYwKhRo+jXr1+j7U6bNo0JEyYwaNAgysrKkiqTPWfOHN5///26TuKysjJ+9atfNRnf3LlzOf300+nUqRPHHntsXfJrjXvvvZcZM2awa9cuevfuzbx584DgPs0XXnghZsbJJ59ct/43v/lNKisrGTZsGO5OcXExixcvbvX+RbJFZahzyNKlS7nxxhv54x//mO1Q6uzcuZP99tsPd+eSSy6hb9++XHrppdkOK2+1le+q5J9EZagLrmlIWubuu++uu1Bu+/btfOtb38p2SCKSZjojEMkgfVclWwrijCDfEpoUHn1HJVdFmgjMbLyZvWZm683sijivl5jZX8zsRTNbambdW7Ofjh07snXrVv1Hk5zl7mzdupWOHTtmOxSRRiIbNWRmRcBtwElAFbDSzJa4e2wBnBuB+9z9XjM7Hvgv4LzGW0use/fuVFVVUV1dnY7QRSLRsWNHundv1W8dkUhFOXx0JLDe3TcAmNmDwCQgNhH0B2rHWD4NtGrsXYcOHequfhURkZaJsmmoG7A5Zr4qXBbrBeDs8PlZQGczO6jhhszsYjNbZWar9KtfRCS9st1Z/CNgrJn9LzAW2ALsabiSu8919zJ3LysuLs50jCIibVqUTUNbgB4x893DZXXc/S3CMwIz2w+Y7O6p3Z1ERERaJMozgpVAXzPrZWb7AFOAJbErmNnBZlYbw4+BeyKMR0RE4ogsEbj7bmAm8DjwCvCwu68xs2vMbGK42peB18xsHXAokMbSbyIikow2cWWxiIgkVhBXFouISOsoEYiIFDglAhGRAqdEICJS4JQIREQKnBKBiEiBUyIQESlwSgQiIgVOiUBEpMApEYiIFDglAhGRAqdEICJS4JQIREQKnBKBiEiBUyIQESlwSgQiIgVOiUBEpMApEYiIFLhIE4GZjTez18xsvZldEef1nmb2tJn9r5m9aGanRRmPiIg0FlkiMLMi4DbgVKA/MNXM+jdY7SqCm9ofBUwBbo8qHhERiS/KM4KRwHp33+DunwIPApMarOPA/uHzLsBbEcYjIiJxRJkIugGbY+arwmWxyoGvmVkV8Cjw3XgbMrOLzWyVma2qrq6OIlYRkYKV7c7iqcB8d+8OnAb8xswaxeTuc929zN3LiouLMx6kiEhbFmUi2AL0iJnvHi6L9Q3gYQB3fxboCBwcYUwiItJAlIlgJdDXzHqZ2T4EncFLGqzzJnACgJkdSZAI1PYjIpJBkSUCd98NzAQeB14hGB20xsyuMbOJ4Wo/BC4ysxeAB4Dp7u5RxSQiIo21j3Lj7v4oQSdw7LKfxDxfC4yJMgYRkVxRXh5MuSbbncUiIgXj6quzHUF8zSYCM9u3diSPmfUzs4lm1iH60EREJBOSOSNYDnQ0s27AE8B5wPwogxIRaSvKy8EsmOCz57nURJRMIjB33wWcDdzu7ucAA6INS0SkbSgvB/dggs+e510iMLOjgWnA/4TLiqILSUREMimZRPB94MfAI+Hwz97A09GGJSLSWC79im6N2bOzHUF81pJh+2Gn8X7uviO6kBIrKyvzVatWZWv3IpJFZp81sUjLmNlqdy+L91oyo4Z+a2b7m9m+wMvAWjO7LN1BiohIdiTTNNQ/PAM4E3gM6EUwckhEJHK5NOom35ummtJs05CZrQGGAr8F5rj7MjN7wd2HZCLAhtQ0JFK4st00lO39pyKlpiHgLqAS2BdYbmYlQNb6CEREJL2aTQTufou7d3P30zywCRiXgdhEROrJxqibXGqaikoyTUNdgNnAceGiZcA17r494tjiUtOQiGRLITcN3QN8CJwbTjuAeekLT0REsimZMtR93H1yzPzVZlYRVUAiIrkqVy8IS1UyZwQfm9kxtTNmNgb4OLqQRERyU1vqF4iVzBnBt4F7w74CA94HpkcZlIiIZE6zicDdK4AhZrZ/OK+hoyIibUiTicDMftDEcgDc/abmNm5m44FfElQr/ZW7X9/g9Zv5bChqJ+AQd++aVOQiIpIWic4IOqeyYTMrAm4DTgKqgJVmtiS8TzEA7n5pzPrfBY5KZZ8iItJyTSYCd0/17pojgfXuvgHAzB4EJgFrm1h/KsH1CiIikkFR3ry+G7A5Zr4qXNZIWLaiF/BUE69fbGarzGxVdXV12gMVESlkUSaClpgCLHL3PfFedPe57l7m7mXFxcUZDk1E2oq2OvwzVVEmgi1Aj5j57uGyeKYAD0QYi4gIV6fa4N1GNTt81Mw+B0wGSmPXd/drmnnrSqCvmfUiSABTgP8TZ/tHAAcAzyYdtYiIpE0yZwR/IOjk3Q18FDMl5O67gZnA48ArwMPhPY+vMbOJMatOAR70ltwzU0QkSYVQPTRVyVQffdndB2Yonmap+qiItFY+Vw9NVarVR/9mZoPSHJOIiOSIZGoNHQNMN7ONwL8I6g25uw+ONDIRkTRrq9VDU5VMIjg18ihERDJA/QLxJXOryk1AV2BCOHUNl4mISBvQbCIws1nAAuCQcLo/rAskIiJtQDJNQ98ARrn7RwBm9jOCMf+3RhmYiIhkRjKjhgyILf2wJ1wmIiJtQDJnBPOAv5vZI+H8mcCvowtJREQyKZk7lN1kZksJhpECXODu/xtpVCIikjGJ7lC2v7vvMLMDgcpwqn3tQHd/P/rwREQkaonOCH4LnAGsBmIvyrZwvneEcYmISIYkukPZGeFjr8yFIyIimZbMdQR/SWaZiIjkpyYTgZl1DPsHDjazA8zswHAqpYlbToqIJKISD7kp0RnBtwj6B44IH2unPwBzog9NRNoa3SEsNyXqI/gl8Esz+6676ypiEZE2Kpmic7ea2UAzO9fMvl47ZSI4Ecl/ukNY7kvmDmWzgS8D/YFHCcpSr3D3r0QeXRy6Q5lI/irkO4RlW6p3KPsKcALwjrtfAAwBuiS54/Fm9pqZrTezK5pY51wzW2tma8zst8lsV0RE0ieZWkMfu/teM9ttZvsD7wI9mnuTmRUBtwEnAVXASjNb4u5rY9bpC/wYGOPuH5jZIa36K0QkL+gOYbkpmTOCVWbWFbibYNTQ8wRlqJszEljv7hvc/VPgQWBSg3UuAm5z9w8A3P3dpCMXkbyjfoHWWbAASkuhXbvgccGC9G4/mc7i77j7Nne/k+DX/flhE1FzugGbY+araHz9QT+gn5k9Y2bPmdn4eBsys4vNbJWZraqurk5i1yIiuSOVA/mCBXDxxbBpU9C/smlTMJ/OZJCo6NywRK+5+/Np2n9fgs7o7sByMxvk7ttiV3L3ucBcCDqL07BfEZGMqD2Q79oVzNceyAGmTWv+/Vde+dl7a+3aFSxP5v3JSNRH8IvwsSNQBrxAUHBuMLAKOLqZbW+hfl9C93BZrCrg7+5eA2w0s3UEiWFlUtGLiOS4VA/kb77ZsuWt0WTTkLuPc/dxwNvAMHcvc/fhwFE0PqDHsxLoa2a9zGwfYAqwpME6iwnOBjCzgwmaija0+K8QEclRqR7Ie/Zs2fLWSKaz+HB3f6l2xt1fBo5s7k3uvhuYCTwOvAI87O5rzOwaM5sYrvY4sNXM1gJPA5e5+9aW/hEikhnq7G25VA/k110HnTrVX9apU7A8XZK5oOwB4CPg/nDRNGA/d5+avjCSpwvKRLJHF4S1XMM+AggO5HPnJt/Gv2BB0JT05ptBArnuupb3D6R6QdkFwBpgVjitDZeJiBSEVEb9TJsWHPRLSoJEWlLSsiRQu43KSti7N3hMVydxrWbPCHKNzghEMqu8PH7V0NmzC6OpKB2/6HNBojOCJhOBmT3s7uea2UvUv1UlAO4+OL1hJkeJQCR7CrFpqLQ0GPLZUElJ8Os8XyRKBImGj84KH89If0giIvkhE8M3sy3R8NG3w8dN8abMhSgi6ZJqU04h1grKxPDNbEt0q8oPzWxHnOlDM9uRySBFJD1SvUNYvvYJpNLZm4nhm9mW6A5lnTMZiIhIFFIt8VC7TqrDN3NZMsNHATCzQ8ysZ+0UZVAikj6FfoewRCUekhX18M1sS+aCsokEdYe+SHAvghLgFXcfEH14jWnUkEjrFeKon3bt4v/NZsGBvVCkekHZtcBoYJ279yK4W9lzaYxPRCQyhdDZm6pkEkFNWP+nnZm1c/enCaqRikieKcRRP4XQ2ZuqZBLBNjPbD1gOLDCzXxLUHhKRPJOv/QLZLvHQ1iXTR7Av8AnBvQimEdy4fkG2qoSqj0CksLSVEg/Z1qo+AjO7zczGuPtH7r7H3Xe7+73ufotKRYtkR77+ok9FOkb9SGKJmobWATeaWaWZ3WBmR2UqKBGJL9ULwvJRIZR4yLZEJSZ+6e5HA2OBrcA9Zvaqmc02s34Zi1BECppG/USv2c7isLbQz9z9KGAqcCbBHcdEJAMK/YIwjfqJXrOJwMzam9kEM1sAPAa8BpwdeWQiAgQHfPfPLoqqfV4oiUCjfqKXqLP4JDO7B6gCLgL+B+jj7lPc/Q/JbNzMxpvZa2a23syuiPP6dDOrNrOKcPpma/8QEWm72nqJh2xLdD+CHwO/BX7o7h+0dMNmVgTcBpxEkExWmtkSd1/bYNWH3H1mS7cvUogK8YIwiV6i6qPHp7jtkcB6d98AYGYPApMI7nksIq1QKM1BkllJVx9thW7A5pj5qnBZQ5PN7EUzW2RmPeJtyMwuNrNVZraquro6ilhFRApWlIkgGf8PKA3vf/wkcG+8ldx9rruXuXtZcXFxRgMUkdRKPKTj/RKtRH0EqdoCxP7C7x4uq9PgCuVfATdEGI+ItEKqN3ZJ9f0SvSjPCFYCfc2sl5ntA0wBlsSuYGaHxcxORNcnSBuWr+37qZZ4UImI3BdZInD33cBM4HGCA/zD7r7GzK4Jb3YD8D0zW2NmLwDfA6ZHFY9ItuVreYhUSzyoRETui7JpCHd/FHi0wbKfxDz/McEwVRHJUT17Bs058ZZn4v0SvWx3FoskLR+bVtpCeYhUSzyoRETua/Z+BLlG9yMoXPl+v918jn/BgqBN/803g1/y113Xso7eVN8vqUt0PwIlAskb+XwghfyPX/JbqjevF8mattC0UkvlISRX6YxA8oZ+UYu0ns4IRESkSUoEkjey3bSSj81RtVTiQRJR05BIkvK1aaphiQcIhm/q5i6FRU1DIgVMJR6kOUoEIgm0hVFLKvEgzVEiEEmgLdwvuKlSDirxILWUCETyQCqdvSrxIM1RIpCCkeqv+GyNWqrt7N20KTgbqa3nn2wymDYt6BguKQmatUpK1FEs9WnUkBSMfB31U1oav3pnSQlUVmY6GslXGjUkksfU2StRUyKQNq0tjPpRZ69ETYlA2rS2MOpHnb0SNSUCkRynzl6JWqSJwMzGm9lrZrbezK5IsN5kM3Mzi9uRkSrVWRHIfq2iVEybFnQM790bPCoJSDpFlgjMrAi4DTgV6A9MNbP+cdbrDMwC/h5FHKkOvZO2I1vNQfohIrkuyjOCkcB6d9/g7p8CDwKT4qx3LfAz4JMoglCdFckm/RCRfBBlIugGbI6ZrwqX1TGzYUAPd/+fRBsys4vNbJWZraqurm5REBp6J9mkHyKSD7LWWWxm7YCbgB82t667z3X3MncvKy4ubtF+NPROskk/RCQfRJkItgA9Yua7h8tqdQYGAkvNrBIYDSxJd4exht5JNumHiOSDKBPBSqCvmfUys32AKcCS2hfdfbu7H+zupe5eCjwHTHT3tNaP0NA7ySb9EJF8EFkicPfdwEzgceAV4GF3X2Nm15jZxKj2G4+G3kkqUhn1ox8ikg9UdE4kAd3mUdoKFZ0TaSWN+pFCoEQgGZOPF3Rp1I8UAiUCyZirr878PlO9oEujfqQQKBFIm5Zq045G/UghUCKQSGX7fgCpNu1o1I8UAo0akozJxq0idZtHkYBGDUla5NPNXGqpaUekeUoEkrRUO3tbez8AXdAlEi01DUnSstG0owu6RNJDTUPSatnu7NUFXSLRK6hEkI9t3NmW7Zu/64IukegVVCLIxgVNkhpd0CUSvYJKBIUu1V/x2bj5u0b9iESvzSeCbLdx55JUz4ha+5lp1I9IbiuoUUPZGPWSSzTqR6RwadRQAcv2GZFG/YjkvoJKBNm4oCnbNOpHRJpTUImgNQe/VMsYFzqN+hHJfZEmAjMbb2avmdl6M7sizuszzOwlM6swsxVm1j/KeFqjLTVtZOOMSKN+RPKAu0cyAUXAG0BvYB/gBaB/g3X2j3k+EfhTc9sdPny4Z5JZbWNK/ckso2G4u/vs2Znf5/33u3fqVP9v79QpWN6SbZSUBJ9ZSUnL3isi6QGs8iaOq+W8/s4AAAhQSURBVJGNGjKzo4Fydz8lnP9xmHj+q4n1pwJfd/dTE20307WGcqmMsco4i0hrZWvUUDdgc8x8VbisHjO7xMzeAG4AvhdvQ2Z2sZmtMrNV1dXVkQTblEJv2lBnr0jbl/XOYne/zd37AJcDVzWxzlx3L3P3suLi4ozGl+0LmrI9/FOdvSJtX5SJYAvQI2a+e7isKQ8CZ0YYT6tNmxY0g/zkJ8FjS5NAbWerWcs7W9Mx/FOdvSKSUFOdB6lOQHtgA9CLzzqLBzRYp2/M8wkk6MyonTLdWRwLWv6edHa2Qss7W9XZKyLuWeosBjCz04D/JhhBdI+7X2dm14QBLTGzXwInAjXAB8BMd1+TaJvZvDFNazprU+1sTbVEgzp7RQQSdxYXVK2h1igvj1+sbfbs5Jpnatv240nmo0/1QN6uXfz9mMHevc2/X0TaBtUaSkGqbfQlJS1b3lCqo3bU2SsizVEiiFiqna2pHsjV2SsizVEiaIHWlGiIHX4KLR9+muqBPNvDX0Uk96mPIA8sWBDUNnrzzeBM4LrrdCAXkZZJ1EfQPtPBSMtNm6YDv4hER01DIiIFTolARKTAKRGIiBQ4JQIRkQKnRCAiUuDybviomVUDcYou5ISDgfeyHUQCii81uR4f5H6Mii81qcRX4u5x6/jnXSLIZWa2qqlxurlA8aUm1+OD3I9R8aUmqvjUNCQiUuCUCERECpwSQXrNzXYAzVB8qcn1+CD3Y1R8qYkkPvURiIgUOJ0RiIgUOCUCEZECp0TQQmbWw8yeNrO1ZrbGzGbFWefLZrbdzCrC6ScZjrHSzF4K992oZrcFbjGz9Wb2opkNy2Bsh8d8LhVmtsPMvt9gnYx/fmZ2j5m9a2Yvxyw70MyeNLPXw8cDmnjv+eE6r5vZ+RmK7edm9mr47/eImXVt4r0JvwsRx1huZlti/h1Pa+K9483stfD7eEUG43soJrZKM6to4r2RfoZNHVMy+v1r6q72muJPwGHAsPB5Z2Ad0L/BOl8G/pjFGCuBgxO8fhrwGGDAaODvWYqzCHiH4EKXrH5+wHHAMODlmGU3AFeEz68AfhbnfQcCG8LHA8LnB2QgtpOB9uHzn8WLLZnvQsQxlgM/SuI78AbQG9gHeKHh/6eo4mvw+i+An2TjM2zqmJLJ75/OCFrI3d929+fD5x8CrwDdshtVi00C7vPAc0BXMzssC3GcALzh7lm/UtzdlwPvN1g8Cbg3fH4vcGact54CPOnu77v7B8CTwPioY3P3J9x9dzj7HNA9nftsqSY+v2SMBNa7+wZ3/xR4kOBzT6tE8ZmZAecCD6R7v8lIcEzJ2PdPiSAFZlYKHAX8Pc7LR5vZC2b2mJkNyGhg4MATZrbazC6O83o3YHPMfBXZSWZTaPo/XzY/v1qHuvvb4fN3gEPjrJMLn+WFBGd48TT3XYjazLD56p4mmjZy4fM7Fvinu7/exOsZ+wwbHFMy9v1TImglM9sP+B3wfXff0eDl5wmaO4YAtwKLMxzeMe4+DDgVuMTMjsvw/ptlZvsAE4GFcV7O9ufXiAfn4Tk31trMrgR2AwuaWCWb34U7gD7AUOBtguaXXDSVxGcDGfkMEx1Tov7+KRG0gpl1IPgHW+Duv2/4urvvcPed4fNHgQ5mdnCm4nP3LeHju8AjBKffsbYAPWLmu4fLMulU4Hl3/2fDF7L9+cX4Z22TWfj4bpx1svZZmtl04AxgWnigaCSJ70Jk3P2f7r7H3fcCdzex76x+F82sPXA28FBT62TiM2zimJKx758SQQuF7Ym/Bl5x95uaWOcL4XqY2UiCz3lrhuLb18w61z4n6FR8ucFqS4Cvh6OHRgPbY05BM6XJX2HZ/PwaWALUjsI4H/hDnHUeB042swPCpo+Tw2WRMrPxwP8FJrr7ribWSea7EGWMsf1OZzWx75VAXzPrFZ4lTiH43DPlROBVd6+K92ImPsMEx5TMff+i6glvqxNwDMEp2otARTidBswAZoTrzATWEIyAeA74twzG1zvc7wthDFeGy2PjM+A2gtEaLwFlGf4M9yU4sHeJWZbVz48gKb0N1BC0s34DOAj4C/A68GfgwHDdMuBXMe+9EFgfThdkKLb1BG3Dtd/BO8N1vwg8mui7kMHP7zfh9+tFgoPaYQ1jDOdPIxgp80ZUMcaLL1w+v/Z7F7NuRj/DBMeUjH3/VGJCRKTAqWlIRKTAKRGIiBQ4JQIRkQKnRCAiUuCUCERECpwSgUjIzPZY/cqoaauEaWalsZUvRXJJ+2wHIJJDPnb3odkOQiTTdEYg0oywHv0NYU36f5jZl8LlpWb2VFhU7S9m1jNcfqgF9wh4IZz+LdxUkZndHdacf8LMPh+u/72wFv2LZvZglv5MKWBKBCKf+XyDpqGvxry23d0HAXOA/w6X3Qrc6+6DCYq+3RIuvwVY5kHRvGEEV6QC9AVuc/cBwDZgcrj8CuCocDszovrjRJqiK4tFQma20933i7O8Ejje3TeExcHecfeDzOw9grIJNeHyt939YDOrBrq7+79itlFKUDe+bzh/OdDB3X9qZn8CdhJUWV3sYcE9kUzRGYFIcryJ5y3xr5jne/isj+50gtpPw4CVYUVMkYxRIhBJzldjHp8Nn/+NoFomwDTgr+HzvwDfBjCzIjPr0tRGzawd0MPdnwYuB7oAjc5KRKKkXx4in/m81b+B+Z/cvXYI6QFm9iLBr/qp4bLvAvPM7DKgGrggXD4LmGtm3yD45f9tgsqX8RQB94fJwoBb3H1b2v4ikSSoj0CkGWEfQZm7v5ftWESioKYhEZECpzMCEZECpzMCEZECp0QgIlLglAhERAqcEoGISIFTIhARKXD/H+BAjl8YrGAOAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "dpt_model_val_loss = dpt_model_hist.history['val_loss']\n", "\n", "plt.plot(epochs, original_val_loss, 'b+', label='Original model')\n", "plt.plot(epochs, dpt_model_val_loss, 'bo', label='Dropout-regularized model')\n", "plt.xlabel('Epochs')\n", "plt.ylabel('Validation loss')\n", "plt.legend()\n", "\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": { "slideshow": { "slide_type": "slide" } }, "source": [ "## Exercise 4: Word embeddings\n", "\n", "* Instead of one-hot-encoding, use a word embedding of length 300\n", "* Only add an output layer after the embedding.\n", "* Evaluate as before. Does it perform better?" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Model: \"sequential_39\"\n", "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "embedding (Embedding) (None, 20, 300) 3000000 \n", "_________________________________________________________________\n", "flatten (Flatten) (None, 6000) 0 \n", "_________________________________________________________________\n", "dense_119 (Dense) (None, 1) 6001 \n", "=================================================================\n", "Total params: 3,006,001\n", "Trainable params: 3,006,001\n", "Non-trainable params: 0\n", "_________________________________________________________________\n", "None\n" ] } ], "source": [ "from tensorflow.keras.layers import Embedding, Flatten, Dense\n", "\n", "max_length = 20 # pad documents to a maximum number of words\n", "vocab_size = 10000 # vocabulary size\n", "embedding_length = 300 # vocabulary size\n", "# define the model\n", "model = models.Sequential()\n", "model.add(Embedding(vocab_size, embedding_length, input_length=max_length))\n", "model.add(Flatten())\n", "model.add(Dense(1, activation='sigmoid'))\n", "# compile the mode\n", "model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n", "# summarize the model\n", "print(model.summary())" ] } ], "metadata": { "anaconda-cloud": {}, "celltoolbar": "Slideshow", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" } }, "nbformat": 4, "nbformat_minor": 4 }